diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index 5f61739f4d8..25017a17122 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1611,7 +1611,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"labels-applied","question":"Did the agent apply at least one label to an unlabeled issue, or correctly call noop when no unlabeled issues were found?"},{"id":"report-created","question":"Was a summary discussion created listing the issues processed and the labels applied?"}]' - GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_MODEL: "copilot/gpt-5.4" GH_AW_EVALS_PHASE: setup with: script: | @@ -1690,7 +1690,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"labels-applied","question":"Did the agent apply at least one label to an unlabeled issue, or correctly call noop when no unlabeled issues were found?"},{"id":"report-created","question":"Was a summary discussion created listing the issues processed and the labels applied?"}]' - GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_MODEL: "copilot/gpt-5.4" GH_AW_EVALS_PHASE: parse with: script: | @@ -1699,17 +1699,27 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); await main(); - name: Redact secrets in evals results + id: redact_evals_results if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); await main(); - name: Upload evals results - if: always() + if: steps.redact_evals_results.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: evals diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index efb243882a8..e1f567a4f4f 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1744,7 +1744,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: small COPILOT_SDK_URI: http://127.0.0.1:3002 GH_AW_COPILOT_SDK_DRIVER: 1 GH_AW_COPILOT_SDK_SERVER_ARGS: '["--headless","--no-auto-update","--port","3002","--add-dir","/tmp/gh-aw/","--log-level","all","--log-dir","/tmp/gh-aw/sandbox/agent/logs/","--disable-builtin-mcps","--no-ask-user","--allow-all-tools"]' @@ -1786,17 +1786,27 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); await main(); - name: Redact secrets in evals results + id: redact_evals_results if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); await main(); - name: Upload evals results - if: always() + if: steps.redact_evals_results.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: evals diff --git a/.github/workflows/contribution-check.lock.yml b/.github/workflows/contribution-check.lock.yml index c40d201d790..ac03dd61c0c 100644 --- a/.github/workflows/contribution-check.lock.yml +++ b/.github/workflows/contribution-check.lock.yml @@ -1793,7 +1793,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: small GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} @@ -1830,17 +1830,27 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); await main(); - name: Redact secrets in evals results + id: redact_evals_results if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); await main(); - name: Upload evals results - if: always() + if: steps.redact_evals_results.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: evals diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 75768bf2e07..acef6e3a0e3 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -2876,7 +2876,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"smoke-passed","question":"Did all or most smoke tests pass? Look for an overall PASS status or the majority of tests showing ✅ in the agent output."},{"id":"build-succeeded","question":"Did the gh-aw binary build succeed? Look for a successful make build step or explicit mention that compilation succeeded."},{"id":"issue-created","question":"Was a smoke test issue created with test results? Look for a create_issue output containing ''Smoke Test'' in the title."}]' - GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_MODEL: "gpt-5.4" GH_AW_EVALS_PHASE: setup with: script: | @@ -2973,7 +2973,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_EVALS_QUESTIONS: '[{"id":"smoke-passed","question":"Did all or most smoke tests pass? Look for an overall PASS status or the majority of tests showing ✅ in the agent output."},{"id":"build-succeeded","question":"Did the gh-aw binary build succeed? Look for a successful make build step or explicit mention that compilation succeeded."},{"id":"issue-created","question":"Was a smoke test issue created with test results? Look for a create_issue output containing ''Smoke Test'' in the title."}]' - GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_MODEL: "gpt-5.4" GH_AW_EVALS_PHASE: parse with: script: | @@ -2982,17 +2982,27 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); await main(); - name: Redact secrets in evals results + id: redact_evals_results if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' continue-on-error: true uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); - const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_secrets.cjs'); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); await main(); - name: Upload evals results - if: always() + if: steps.redact_evals_results.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: evals diff --git a/actions/setup/js/evals_constants.cjs b/actions/setup/js/evals_constants.cjs new file mode 100644 index 00000000000..30e75a627cd --- /dev/null +++ b/actions/setup/js/evals_constants.cjs @@ -0,0 +1,5 @@ +"use strict"; + +const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; + +module.exports = { EVALS_OUTPUT_PATH }; diff --git a/actions/setup/js/redact_evals_results.cjs b/actions/setup/js/redact_evals_results.cjs new file mode 100644 index 00000000000..d7a0a15d7ba --- /dev/null +++ b/actions/setup/js/redact_evals_results.cjs @@ -0,0 +1,46 @@ +// @ts-check +/// + +const fs = require("fs"); +const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs"); +const { main: redactWorkspaceSecrets, redactSecrets, redactBuiltInPatterns, extractMCPGatewayTokens, MCP_GATEWAY_CONFIG_PATHS } = require("./redact_secrets.cjs"); + +function getSecretValues() { + const secretNames = (process.env.GH_AW_SECRET_NAMES || "") + .split(",") + .map(name => name.trim()) + .filter(Boolean); + + /** @type {string[]} */ + const secretValues = []; + for (const secretName of secretNames) { + const value = process.env[`SECRET_${secretName}`]; + if (typeof value === "string" && value.trim() !== "") { + secretValues.push(value.trim()); + } + } + + secretValues.push(...extractMCPGatewayTokens(MCP_GATEWAY_CONFIG_PATHS)); + return secretValues; +} + +function verifyRedaction() { + if (!fs.existsSync(EVALS_OUTPUT_PATH)) { + return; + } + + const content = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8"); + const secretValues = getSecretValues(); + const lingeringRedactions = redactBuiltInPatterns(content).redactionCount + redactSecrets(content, secretValues).redactionCount; + + if (lingeringRedactions > 0) { + core.setFailed(`Secret redaction verification failed for ${EVALS_OUTPUT_PATH}: ${lingeringRedactions} unredacted value(s) remain`); + } +} + +async function main() { + await redactWorkspaceSecrets(); + verifyRedaction(); +} + +module.exports = { main, getSecretValues, verifyRedaction }; diff --git a/actions/setup/js/redact_evals_results.test.cjs b/actions/setup/js/redact_evals_results.test.cjs new file mode 100644 index 00000000000..4bca9f906a4 --- /dev/null +++ b/actions/setup/js/redact_evals_results.test.cjs @@ -0,0 +1,89 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "fs"; + +const OUTPUT_DIR = "/tmp/gh-aw"; +const OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; +const MCP_CONFIG_DIR = `${process.env.RUNNER_TEMP || "/tmp"}/gh-aw/mcp-config`; +const GATEWAY_OUTPUT_PATH = `${MCP_CONFIG_DIR}/gateway-output.json`; + +const mockCore = { + info: vi.fn(), + warning: vi.fn(), + setFailed: vi.fn(), +}; + +global.core = mockCore; + +describe("redact_evals_results.cjs", () => { + let module; + + beforeEach(async () => { + vi.clearAllMocks(); + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + if (fs.existsSync(OUTPUT_PATH)) { + fs.unlinkSync(OUTPUT_PATH); + } + delete process.env.GH_AW_SECRET_NAMES; + delete process.env.SECRET_EVALS_SECRET; + module = await import("./redact_evals_results.cjs"); + }); + + afterEach(() => { + if (fs.existsSync(OUTPUT_PATH)) { + fs.unlinkSync(OUTPUT_PATH); + } + if (fs.existsSync(GATEWAY_OUTPUT_PATH)) { + fs.unlinkSync(GATEWAY_OUTPUT_PATH); + } + }); + + it("collects eval secret values from the workflow env", () => { + process.env.GH_AW_SECRET_NAMES = "EVALS_SECRET, EMPTY_SECRET"; + process.env.SECRET_EVALS_SECRET = "secret-value"; + process.env.SECRET_EMPTY_SECRET = " "; + + expect(module.getSecretValues()).toContain("secret-value"); + expect(module.getSecretValues()).not.toContain(""); + }); + + it("collects MCP gateway tokens from the canonical config path", () => { + const tokenPart = "gateway-token-abc123"; + const bearerToken = ["Bearer", tokenPart].join(" "); + fs.mkdirSync(MCP_CONFIG_DIR, { recursive: true }); + fs.writeFileSync( + GATEWAY_OUTPUT_PATH, + JSON.stringify({ + mcpServers: { + github: { + headers: { Authorization: bearerToken }, + }, + }, + }), + "utf8" + ); + + expect(module.getSecretValues()).toContain(bearerToken); + expect(module.getSecretValues()).toContain(tokenPart); + }); + + it("redacts eval output and leaves no lingering secrets", async () => { + process.env.GH_AW_SECRET_NAMES = "EVALS_SECRET"; + process.env.SECRET_EVALS_SECRET = "super-secret-value"; + fs.writeFileSync(OUTPUT_PATH, `{"question":"Contains super-secret-value"}\n`, "utf8"); + + await module.main(); + + expect(fs.readFileSync(OUTPUT_PATH, "utf8")).toContain("***REDACTED***"); + expect(mockCore.setFailed).not.toHaveBeenCalled(); + }); + + it("fails closed when verification still detects secrets", () => { + process.env.GH_AW_SECRET_NAMES = "EVALS_SECRET"; + process.env.SECRET_EVALS_SECRET = "super-secret-value"; + fs.writeFileSync(OUTPUT_PATH, `{"question":"Contains super-secret-value"}\n`, "utf8"); + + module.verifyRedaction(); + + expect(mockCore.setFailed).toHaveBeenCalledWith(expect.stringContaining("Secret redaction verification failed")); + }); +}); diff --git a/actions/setup/js/render_evals_summary.cjs b/actions/setup/js/render_evals_summary.cjs new file mode 100644 index 00000000000..7ca60b61c29 --- /dev/null +++ b/actions/setup/js/render_evals_summary.cjs @@ -0,0 +1,127 @@ +// @ts-check +/// + +/** + * render_evals_summary — BinEval results step-summary renderer. + * + * Reads the redacted evals.jsonl written by run_evals (parse phase) and + * renders the results as a collapsible
section in the GitHub + * Actions step summary. + * + * This step runs AFTER redact_secrets so that no credentials appear in the + * published summary. It is a no-op when the file does not exist or is empty. + */ + +"use strict"; + +const fs = require("fs"); +const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs"); +const { buildStepSummaryDetailsSection } = require("./log_parser_step_summary_builder.cjs"); + +/** + * Reads and parses evals.jsonl records. + * Returns an empty array when the file is absent or unparseable. + * @returns {Array<{id: string, question: string, answer: string, model: string, timestamp: string}>} + */ +function readEvalsResults() { + if (!fs.existsSync(EVALS_OUTPUT_PATH)) { + return []; + } + + const results = []; + let raw; + try { + raw = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8"); + } catch { + return []; + } + + for (const rawLine of raw.split("\n")) { + const line = rawLine.trim(); + if (!line) continue; + try { + const record = JSON.parse(line); + if (record && typeof record === "object") { + results.push({ + id: String(record.id ?? ""), + question: String(record.question ?? ""), + answer: String(record.answer ?? "UNKNOWN") + .trim() + .toUpperCase(), + model: String(record.model ?? ""), + timestamp: String(record.timestamp ?? ""), + }); + } + } catch { + // skip malformed lines + } + } + + return results; +} + +/** + * Builds the markdown body for the evals
section. + * @param {Array<{id: string, question: string, answer: string, model: string, timestamp: string}>} results + * @returns {string} + */ +function buildEvalsBody(results) { + if (results.length === 0) { + return ""; + } + + const yesCount = results.filter(r => r.answer === "YES").length; + const noCount = results.filter(r => r.answer === "NO").length; + const unknownCount = results.filter(r => r.answer === "UNKNOWN").length; + + const lines = []; + lines.push(`| ID | Question | Answer |`); + lines.push(`| --- | --- | --- |`); + for (const r of results) { + const answerEmoji = r.answer === "YES" ? "✅ YES" : r.answer === "NO" ? "❌ NO" : "❓ UNKNOWN"; + lines.push(`| ${escapeMarkdownCell(r.id)} | ${escapeMarkdownCell(r.question)} | ${answerEmoji} |`); + } + lines.push(""); + lines.push(`**YES**: ${yesCount} | **NO**: ${noCount} | **UNKNOWN**: ${unknownCount}`); + + const model = results[0]?.model; + if (model) { + lines.push(`**model**: ${escapeMarkdownCell(model)}`); + } + + return lines.join("\n"); +} + +/** + * Escapes a string for use inside a Markdown table cell. + * @param {string} text + * @returns {string} + */ +function escapeMarkdownCell(text) { + return text + .replace(/[\r\n]/g, " ") + .replace(/\|/g, "\\|") + .replace(/`/g, "\\`"); +} + +/** + * Main entry point: reads evals results and writes the step summary section. + * @returns {Promise} + */ +async function main() { + const results = readEvalsResults(); + if (results.length === 0) { + core.info("No evals results found; skipping step summary section"); + return; + } + + core.info(`Rendering evals summary: ${results.length} result(s)`); + + const body = buildEvalsBody(results); + const markdown = buildStepSummaryDetailsSection("BinEval Results", body); + + await core.summary.addRaw(markdown).write(); + core.info("BinEval results section written to step summary"); +} + +module.exports = { main, readEvalsResults, buildEvalsBody }; diff --git a/actions/setup/js/render_evals_summary.test.cjs b/actions/setup/js/render_evals_summary.test.cjs new file mode 100644 index 00000000000..ad5f00753b2 --- /dev/null +++ b/actions/setup/js/render_evals_summary.test.cjs @@ -0,0 +1,148 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "fs"; + +const OUTPUT_DIR = "/tmp/gh-aw"; +const OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; + +const mockCore = { + info: vi.fn(), + summary: { + addRaw: vi.fn().mockReturnThis(), + write: vi.fn().mockResolvedValue(), + }, +}; + +global.core = mockCore; + +describe("render_evals_summary.cjs", () => { + let module; + + beforeEach(async () => { + vi.clearAllMocks(); + fs.mkdirSync(OUTPUT_DIR, { recursive: true }); + if (fs.existsSync(OUTPUT_PATH)) { + fs.unlinkSync(OUTPUT_PATH); + } + module = await import("./render_evals_summary.cjs"); + }); + + afterEach(() => { + if (fs.existsSync(OUTPUT_PATH)) { + fs.unlinkSync(OUTPUT_PATH); + } + }); + + describe("readEvalsResults", () => { + it("returns an empty array when the output file does not exist", () => { + expect(module.readEvalsResults()).toEqual([]); + }); + + it("parses JSONL, skips malformed lines, and normalizes answers", () => { + fs.writeFileSync( + OUTPUT_PATH, + [ + JSON.stringify({ id: "q1", question: "First?", answer: " yes ", model: "claude-sonnet-4.6", timestamp: "2026-07-15T00:00:00Z" }), + "not-json", + JSON.stringify({ id: "q2", question: "Second?", answer: "No", model: "claude-sonnet-4.6" }), + JSON.stringify({ id: "q3", question: "Third?" }), + ].join("\n"), + "utf8" + ); + + expect(module.readEvalsResults()).toEqual([ + { + id: "q1", + question: "First?", + answer: "YES", + model: "claude-sonnet-4.6", + timestamp: "2026-07-15T00:00:00Z", + }, + { + id: "q2", + question: "Second?", + answer: "NO", + model: "claude-sonnet-4.6", + timestamp: "", + }, + { + id: "q3", + question: "Third?", + answer: "UNKNOWN", + model: "", + timestamp: "", + }, + ]); + }); + }); + + describe("buildEvalsBody", () => { + it("renders tallies, escapes table cells, and includes the model", () => { + const markdown = module.buildEvalsBody([ + { + id: "id|`1`\r", + question: "Line 1\nLine 2 | `code`", + answer: "YES", + model: "claude`-4.6`", + timestamp: "", + }, + { + id: "id2", + question: "Question 2", + answer: "NO", + model: "claude`-4.6`", + timestamp: "", + }, + { + id: "id3", + question: "Question 3", + answer: "UNKNOWN", + model: "claude`-4.6`", + timestamp: "", + }, + ]); + + expect(markdown).toContain("| id\\|\\`1\\` | Line 1 Line 2 \\| \\`code\\` | ✅ YES |"); + expect(markdown).toContain("| id2 | Question 2 | ❌ NO |"); + expect(markdown).toContain("| id3 | Question 3 | ❓ UNKNOWN |"); + expect(markdown).toContain("**YES**: 1 | **NO**: 1 | **UNKNOWN**: 1"); + expect(markdown).toContain("**model**: claude\\`-4.6\\`"); + }); + + it("returns an empty string for empty results", () => { + expect(module.buildEvalsBody([])).toBe(""); + }); + }); + + describe("main", () => { + it("logs and returns early when the output file is empty or malformed", async () => { + fs.writeFileSync(OUTPUT_PATH, "not-json\n", "utf8"); + + await module.main(); + + expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("No evals results found")); + expect(mockCore.summary.addRaw).not.toHaveBeenCalled(); + expect(mockCore.summary.write).not.toHaveBeenCalled(); + }); + + it("writes the evals details section to the step summary", async () => { + fs.writeFileSync( + OUTPUT_PATH, + [JSON.stringify({ id: "builds", question: "Does it build?", answer: "YES", model: "claude-sonnet-4.6" }), JSON.stringify({ id: "tests", question: "Do tests pass?", answer: "NO", model: "claude-sonnet-4.6" })].join("\n") + "\n", + "utf8" + ); + + await module.main(); + + expect(mockCore.summary.addRaw).toHaveBeenCalledTimes(1); + const summary = mockCore.summary.addRaw.mock.calls[0][0]; + expect(summary).toContain("
"); + expect(summary).toContain("BinEval Results"); + expect(summary).toContain("✅ YES"); + expect(summary).toContain("❌ NO"); + expect(summary).toContain("**model**: claude-sonnet-4.6"); + expect(mockCore.summary.write).toHaveBeenCalledTimes(1); + expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("2 result(s)")); + expect(mockCore.info).toHaveBeenCalledWith(expect.stringContaining("written to step summary")); + }); + }); +}); diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index 72c6bf1f601..c379df655ac 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -34,10 +34,10 @@ const fs = require("fs"); const path = require("path"); const { ERR_VALIDATION } = require("./error_codes.cjs"); +const { EVALS_OUTPUT_PATH } = require("./evals_constants.cjs"); const EVALS_DIR = "/tmp/gh-aw/evals"; const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log"; -const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; const AGENT_OUTPUT_FILENAME = "agent_output.json"; // --------------------------------------------------------------------------- @@ -156,23 +156,8 @@ async function parseMain() { const jsonlLines = results.map(r => JSON.stringify(r)); fs.writeFileSync(EVALS_OUTPUT_PATH, jsonlLines.join("\n") + (jsonlLines.length > 0 ? "\n" : "")); core.info(`BinEval results written to ${EVALS_OUTPUT_PATH} (${results.length} record(s))`); - - const yesCount = results.filter(r => r.answer === "YES").length; - const noCount = results.filter(r => r.answer === "NO").length; - const unknownCount = results.filter(r => r.answer === "UNKNOWN").length; - - await core.summary - .addHeading("BinEval Results", 2) - .addTable([ - [ - { data: "ID", header: true }, - { data: "Question", header: true }, - { data: "Answer", header: true }, - ], - ...results.map(r => [r.id, r.question, r.answer]), - ["", `YES: ${yesCount} | NO: ${noCount} | UNKNOWN: ${unknownCount}`, ""], - ]) - .write(); + // Step summary rendering is handled by the dedicated render_evals_summary.cjs step + // that runs after secret redaction, so the published summary is always redacted. } // --------------------------------------------------------------------------- diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index d7fe47a5426..64718732e1f 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -61,7 +61,11 @@ func (c *Compiler) buildEvalsJobSteps(data *WorkflowData) []string { // Step 9: Redact secrets from evals results before upload. steps = append(steps, c.buildRedactEvalsSecretsStep(data)...) - // Step 10: Upload evals.jsonl as the evals artifact. + // Step 10: Render evals results as a progressive disclosure step summary section. + // Runs after redaction so the published summary is always free of secrets. + steps = append(steps, c.buildRenderEvalsSummaryStep(data)...) + + // Step 11: Upload evals.jsonl as the evals artifact. steps = append(steps, c.buildUploadEvalsArtifactStep(data)...) return steps @@ -99,10 +103,7 @@ func (c *Compiler) buildSetupEvalsStep(data *WorkflowData) []string { } questionsJSON := marshalEvalsQuestions(data.Evals.Questions) - model := data.Evals.Model - if model == "" { - model = "small" - } + model := c.resolveEvalsExecutionModel(data) script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); @@ -147,11 +148,6 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { } } - // Override model from evals frontmatter if specified. - if data.Evals != nil && data.Evals.Model != "" { - evalsEngineConfig.Model = data.Evals.Model - } - // Apply engine and enterprise default detection model (cost-effective for Q&A tasks). engine, err := c.getAgenticEngine(engineID) if err != nil { @@ -160,27 +156,12 @@ func (c *Compiler) buildEvalsEngineSteps(data *WorkflowData) []string { } } - if evalsEngineConfig.Model == "" { - if defaultModel := compilerenv.ResolveDefaultDetectionModel(""); defaultModel != "" { - evalsEngineConfig.Model = defaultModel - } else if defaultModel := engine.GetDefaultDetectionModel(); defaultModel != "" { - evalsEngineConfig.Model = defaultModel - } - } - // Inherit APITarget from the main engine config for GHE/custom endpoints. if evalsEngineConfig.APITarget == "" && data.EngineConfig != nil && data.EngineConfig.APITarget != "" { evalsEngineConfig.APITarget = data.EngineConfig.APITarget } - // Normalize Pi engine model to bare model ID for Copilot CLI. - originalEngineID := data.AI - if data.EngineConfig != nil && data.EngineConfig.ID != "" { - originalEngineID = data.EngineConfig.ID - } - if engineID == "copilot" && originalEngineID == "pi" { - evalsEngineConfig.Model = extractPiModelID(evalsEngineConfig.Model) - } + evalsEngineConfig.Model = c.resolveEvalsExecutionModel(data) // Build a minimal WorkflowData for evals engine execution. // IsDetectionRun reuses detection-style network restrictions and MaxAI credits, @@ -278,10 +259,7 @@ func (c *Compiler) buildParseEvalsResultsStep(data *WorkflowData) []string { } questionsJSON := marshalEvalsQuestions(data.Evals.Questions) - model := data.Evals.Model - if model == "" { - model = "small" - } + model := c.resolveEvalsExecutionModel(data) script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); @@ -309,12 +287,50 @@ await main();` func (c *Compiler) buildRedactEvalsSecretsStep(data *WorkflowData) []string { script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); setupGlobals(core, github, context, exec, io, getOctokit); -const { main } = require('` + SetupActionDestination + `/redact_secrets.cjs'); +const { main } = require('` + SetupActionDestination + `/redact_evals_results.cjs'); await main();` + secretReferences := c.collectEvalsSecretReferences(data) + steps := []string{ " - name: Redact secrets in evals results\n", + " id: redact_evals_results\n", " if: always()\n", + fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), + } + if len(secretReferences) > 0 { + steps = append(steps, " env:\n") + escapedRefs := make([]string, len(secretReferences)) + for i, ref := range secretReferences { + escapedRefs[i] = escapeSingleQuoteBackslash(ref) + } + steps = append(steps, fmt.Sprintf(" GH_AW_SECRET_NAMES: '%s'\n", strings.Join(escapedRefs, ","))) + for _, secretName := range secretReferences { + escapedSecretName := escapeSingleQuoteBackslash(secretName) + steps = append(steps, fmt.Sprintf(" SECRET_%s: ${{ secrets.%s }}\n", escapedSecretName, secretName)) + } + } + steps = append(steps, + " with:\n", + " script: |\n", + ) + steps = append(steps, FormatJavaScriptForYAML(script)...) + return steps +} + +// buildRenderEvalsSummaryStep creates a step that reads the redacted evals.jsonl +// and renders the results as a collapsible progressive disclosure section in the +// GitHub Actions step summary. Running after secret redaction ensures no credentials +// appear in the published summary. +func (c *Compiler) buildRenderEvalsSummaryStep(data *WorkflowData) []string { + script := `const { setupGlobals } = require('` + SetupActionDestination + `/setup_globals.cjs'); +setupGlobals(core, github, context, exec, io, getOctokit); +const { main } = require('` + SetupActionDestination + `/render_evals_summary.cjs'); +await main();` + + steps := []string{ + " - name: Render evals results to step summary\n", + " if: steps.redact_evals_results.outcome == 'success'\n", " continue-on-error: true\n", fmt.Sprintf(" uses: %s\n", getCachedActionPin("actions/github-script", data)), " with:\n", @@ -330,7 +346,7 @@ func (c *Compiler) buildUploadEvalsArtifactStep(data *WorkflowData) []string { evalsArtifactName := artifactPrefixExprForDownstreamJob(data) + constants.EvalsArtifactName return []string{ " - name: Upload evals results\n", - " if: always()\n", + " if: steps.redact_evals_results.outcome == 'success'\n", fmt.Sprintf(" uses: %s\n", c.getActionPin("actions/upload-artifact")), " with:\n", " name: " + evalsArtifactName + "\n", @@ -355,6 +371,45 @@ func (c *Compiler) getEvalsEngineID(data *WorkflowData) string { return "copilot" } +func (c *Compiler) resolveEvalsExecutionModel(data *WorkflowData) string { + model := "" + if data.EngineConfig != nil && data.EngineConfig.Model != "" { + model = data.EngineConfig.Model + } + if data.Evals != nil && data.Evals.Model != "" { + model = data.Evals.Model + } + + engineID := c.getEvalsEngineID(data) + if model == "" { + if defaultModel := compilerenv.ResolveDefaultDetectionModel(""); defaultModel != "" { + model = defaultModel + } else if engine, err := c.getAgenticEngine(engineID); err == nil { + model = engine.GetDefaultDetectionModel() + } + } + if model == "" { + model = "small" + } + + originalEngineID := data.AI + if data.EngineConfig != nil && data.EngineConfig.ID != "" { + originalEngineID = data.EngineConfig.ID + } + if engineID == "copilot" && originalEngineID == "pi" { + model = extractPiModelID(model) + } + + return model +} + +func (c *Compiler) collectEvalsSecretReferences(data *WorkflowData) []string { + if data.Evals == nil { + return nil + } + return CollectSecretReferences(marshalEvalsQuestions(data.Evals.Questions) + "\n" + c.resolveEvalsExecutionModel(data)) +} + // --------------------------------------------------------------------------- // Utility helpers // --------------------------------------------------------------------------- diff --git a/pkg/workflow/evals_steps_test.go b/pkg/workflow/evals_steps_test.go index 5627fe46c6e..be018a585fa 100644 --- a/pkg/workflow/evals_steps_test.go +++ b/pkg/workflow/evals_steps_test.go @@ -83,3 +83,113 @@ func TestBuildEvalsEngineStepsArcDindTopology(t *testing.T) { } }) } + +// TestBuildEvalsJobStepsRenderSummary verifies that the evals job includes the +// "Render evals results to step summary" step and that it runs after the redact step. +func TestBuildEvalsJobStepsRenderSummary(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "copilot", + Evals: &EvalsConfig{ + Questions: []EvalDefinition{ + {ID: "builds", Question: "Does the code build?"}, + }, + }, + } + + steps := compiler.buildEvalsJobSteps(data) + if len(steps) == 0 { + t.Fatal("expected non-empty steps") + } + allSteps := strings.Join(steps, "") + + // The render summary step must be present. + if !strings.Contains(allSteps, "- name: Render evals results to step summary") { + t.Errorf("expected 'Render evals results to step summary' step in evals job;\ngot:\n%s", allSteps) + } + if !strings.Contains(allSteps, "id: redact_evals_results") { + t.Errorf("expected redact step id in evals job;\ngot:\n%s", allSteps) + } + if !strings.Contains(allSteps, "redact_evals_results.cjs") { + t.Errorf("expected redact_evals_results.cjs reference in evals job steps;\ngot:\n%s", allSteps) + } + + // The render summary step must call render_evals_summary.cjs. + if !strings.Contains(allSteps, "render_evals_summary.cjs") { + t.Errorf("expected render_evals_summary.cjs reference in evals job steps;\ngot:\n%s", allSteps) + } + if !strings.Contains(allSteps, "if: steps.redact_evals_results.outcome == 'success'") { + t.Errorf("expected redact outcome gating for render/upload steps;\ngot:\n%s", allSteps) + } + + // The render step must appear after the redact step (redact before publish to step summary). + redactIdx := strings.Index(allSteps, "- name: Redact secrets in evals results") + renderIdx := strings.Index(allSteps, "- name: Render evals results to step summary") + uploadIdx := strings.Index(allSteps, "- name: Upload evals results") + if redactIdx < 0 { + t.Error("expected 'Redact secrets in evals results' step") + } + if renderIdx < 0 { + t.Error("expected 'Render evals results to step summary' step") + } + if uploadIdx < 0 { + t.Error("expected 'Upload evals results' step") + } + if redactIdx >= 0 && renderIdx >= 0 && renderIdx <= redactIdx { + t.Errorf("expected render step to appear after redact step; redactIdx=%d renderIdx=%d", redactIdx, renderIdx) + } + if renderIdx >= 0 && uploadIdx >= 0 && renderIdx >= uploadIdx { + t.Errorf("expected render step to appear before upload step; renderIdx=%d uploadIdx=%d", renderIdx, uploadIdx) + } +} + +func TestBuildEvalsJobStepsRedactionUsesEvalsSecretReferences(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "copilot", + Evals: &EvalsConfig{ + Model: "${{ secrets.EVALS_MODEL_SECRET }}", + Questions: []EvalDefinition{ + {ID: "builds", Question: "Does it use ${{ secrets.EVALS_PROMPT_SECRET }}?"}, + }, + }, + } + + steps := strings.Join(compiler.buildRedactEvalsSecretsStep(data), "") + if !strings.Contains(steps, "GH_AW_SECRET_NAMES") { + t.Fatalf("expected GH_AW_SECRET_NAMES in redact step:\n%s", steps) + } + if !strings.Contains(steps, "SECRET_EVALS_MODEL_SECRET: ${{ secrets.EVALS_MODEL_SECRET }}") { + t.Errorf("expected model secret env binding in redact step:\n%s", steps) + } + if !strings.Contains(steps, "SECRET_EVALS_PROMPT_SECRET: ${{ secrets.EVALS_PROMPT_SECRET }}") { + t.Errorf("expected prompt secret env binding in redact step:\n%s", steps) + } +} + +func TestBuildParseEvalsResultsStepUsesResolvedExecutionModel(t *testing.T) { + compiler := NewCompiler() + + data := &WorkflowData{ + AI: "claude", + EngineConfig: &EngineConfig{ + ID: "claude", + Model: "claude-sonnet-4.6", + }, + Evals: &EvalsConfig{ + Questions: []EvalDefinition{ + {ID: "builds", Question: "Does it build?"}, + }, + }, + } + + steps := strings.Join(compiler.buildParseEvalsResultsStep(data), "") + if !strings.Contains(steps, `GH_AW_EVALS_MODEL: "claude-sonnet-4.6"`) { + t.Errorf("expected parse step to record resolved execution model; got:\n%s", steps) + } + if strings.Contains(steps, `GH_AW_EVALS_MODEL: "small"`) { + t.Errorf("expected parse step to avoid default 'small' when engine model is resolved; got:\n%s", steps) + } +}