From 17e78d15728f05ad3fd83bce75f3ede2f9f76c96 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:32:20 +0000 Subject: [PATCH 1/2] Add eval metric summaries to experiments analyze Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/aw/experiments.md | 6 +- .../daily-agentrx-trace-optimizer.lock.yml | 314 ++++++++++++++++- .../daily-agentrx-trace-optimizer.md | 4 + .../dataflow-pr-discussion-dataset.lock.yml | 309 ++++++++++++++++- .../dataflow-pr-discussion-dataset.md | 4 + .github/workflows/deep-report.lock.yml | 314 ++++++++++++++++- .github/workflows/deep-report.md | 4 + .github/workflows/gpclean.lock.yml | 309 ++++++++++++++++- .github/workflows/gpclean.md | 4 + .github/workflows/smoke-antigravity.lock.yml | 299 ++++++++++++++++- .github/workflows/smoke-antigravity.md | 4 + .../smoke-copilot-aoai-apikey.lock.yml | 311 ++++++++++++++++- .../workflows/smoke-copilot-aoai-apikey.md | 6 + .../smoke-copilot-aoai-entra.lock.yml | 316 ++++++++++++++++- .github/workflows/smoke-copilot-aoai-entra.md | 6 + .../smoke-copilot-sub-agents.lock.yml | 317 +++++++++++++++++- .github/workflows/smoke-copilot-sub-agents.md | 4 + .github/workflows/smoke-gemini.lock.yml | 303 ++++++++++++++++- .github/workflows/smoke-gemini.md | 4 + .github/workflows/smoke-project.lock.yml | 309 ++++++++++++++++- .github/workflows/smoke-project.md | 4 + .github/workflows/smoke-temporary-id.lock.yml | 309 ++++++++++++++++- .github/workflows/smoke-temporary-id.md | 4 + .../workflows/test-quality-sentinel.lock.yml | 317 +++++++++++++++++- .github/workflows/test-quality-sentinel.md | 4 + .github/workflows/typist.lock.yml | 314 ++++++++++++++++- .github/workflows/typist.md | 4 + .../content/docs/experimental/experiments.md | 3 + pkg/cli/experiments_analyze_statistics.go | 57 +++- .../experiments_analyze_statistics_test.go | 95 ++++-- pkg/cli/experiments_command.go | 92 ++++- pkg/cli/experiments_command_test.go | 29 ++ 32 files changed, 4324 insertions(+), 55 deletions(-) diff --git a/.github/aw/experiments.md b/.github/aw/experiments.md index 45b64efc78b..00d15daf3dd 100644 --- a/.github/aw/experiments.md +++ b/.github/aw/experiments.md @@ -135,7 +135,7 @@ Use `${{ experiments.tone }}` tone when writing the issue body. 1. **One dimension** per experiment. 2. **Falsifiable hypothesis**. -3. **Primary metric** measurable from workflow run data (artifacts, outputs, duration, tokens). +3. **Primary metric** measurable from workflow run data (artifacts, outputs, duration, tokens). Prefer `eval:` / `evals.` when success is best measured as a YES/NO question. 4. **Guardrail metrics** — things that must not degrade. Use `direction: min` + bare number for lower-is-better rates, or `">=0.95"` for higher-is-better. 5. **Sample size estimate** per variant. @@ -161,6 +161,8 @@ Use `{{#if experiments.prompt_style == "concise" }}` blocks to swap prompt instr **Typical metrics**: output quality, AI credits, success rate, output length. +When `metric` references `eval:` or `evals.`, declare that eval question under `evals:`. `gh aw experiments analyze` will then show both the metric question text and observed eval YES/NO/UNKNOWN results. + ### Engine & Model ```yaml @@ -281,7 +283,7 @@ experiments: 2. **Instrument** — add `experiments:` and `{{#if experiments. == "" }}` blocks. Never use `__GH_AW_EXPERIMENTS__*`. 3. **Compile** — `gh aw compile `. 4. **Run** — check activation job step summary for variant assignment. -5. **Analyse** — once min sample size reached, compare distributions. +5. **Analyse** — once min sample size reached, compare distributions; for eval-backed metrics, use `gh aw experiments analyze ` to inspect the resolved question and current eval outcomes. 6. **Conclude** — rewrite baseline to winning variant, remove `experiments:`, recompile. --- diff --git a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml index ee1755ad26f..e6c158437f7 100644 --- a/.github/workflows/daily-agentrx-trace-optimizer.lock.yml +++ b/.github/workflows/daily-agentrx-trace-optimizer.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"5adce1d918580ec3eaf1eabdfeb27e796d9404c0609868842ad1837f6d572a2a","body_hash":"747e1e939a6ef29e349006d848b54f7acecf2fa7cca5ae09877643ad918968ec","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"70a34c49858efc2f79e68247be89c468c6b0ceda7b2dac7e0573f56b365dfdb4","body_hash":"747e1e939a6ef29e349006d848b54f7acecf2fa7cca5ae09877643ad918968ec","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1244,6 +1244,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs if: > @@ -1311,6 +1313,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1409,6 +1419,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "daily-agentrx-trace-optimizer" with: @@ -1489,6 +1500,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1747,6 +1759,306 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily AgentRx Trace Optimizer" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-agentrx-trace-optimizer.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.220" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dailyagentrxtraceoptimizer-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.220 + - name: Execute Claude Code CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Allowed tools (sorted): + # - Bash + # - BashOutput + # - Edit(/tmp/*) + # - ExitPlanMode + # - Glob + # - Grep + # - KillBash + # - LS + # - MultiEdit(/tmp/*) + # - NotebookRead + # - Read + # - Read(/tmp/*) + # - Task + # - TodoWrite + # - Write(/tmp/*) + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.githubusercontent.com\",\"anthropic.com\",\"api.anthropic.com\",\"api.github.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"cdn.playwright.dev\",\"codeload.github.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"files.pythonhosted.org\",\"ghcr.io\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"playwright.download.prss.microsoft.com\",\"ppa.launchpad.net\",\"pypi.org\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"sentry.io\",\"statsig.anthropic.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --tty --env-all --exclude-env ANTHROPIC_API_KEY --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_MAX_RETRIES: 0 + ANTHROPIC_MODEL: small + BASH_DEFAULT_TIMEOUT_MS: 60000 + BASH_MAX_TIMEOUT_MS: 60000 + CLAUDE_CODE_DISABLE_FAST_MODE: 1 + DISABLE_BUG_COMMAND: 1 + DISABLE_ERROR_REPORTING: 1 + DISABLE_TELEMETRY: 1 + GH_AW_LLM_PROVIDER: anthropic + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + MCP_TIMEOUT: 120000 + MCP_TOOL_TIMEOUT: 60000 + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Daily AgentRx Trace Optimizer" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/daily-agentrx-trace-optimizer.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.220" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/dailyagentrxtraceoptimizer + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/daily-agentrx-trace-optimizer.md b/.github/workflows/daily-agentrx-trace-optimizer.md index 528aaaced48..afe317588ff 100644 --- a/.github/workflows/daily-agentrx-trace-optimizer.md +++ b/.github/workflows/daily-agentrx-trace-optimizer.md @@ -53,6 +53,10 @@ imports: expires: 7d features: gh-aw-detection: true +evals: + - id: sub_agent_strategy_goal_met + question: Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed? + --- {{#runtime-import? .github/shared-instructions.md}} diff --git a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml index b7118918ad8..0621e4b2aef 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.lock.yml +++ b/.github/workflows/dataflow-pr-discussion-dataset.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cbdd289a6d9258b11c33d8d52c475584519e0b91fb42de2310cf87e2f2a6ca32","body_hash":"70341e53871b3268cc556e830b07973a73aea98d4a690fa199321c6fa8e22e70","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"f0b26fe22373938d58762fc148cd7d21e47374299c3275576cea541127cb7d59","body_hash":"70341e53871b3268cc556e830b07973a73aea98d4a690fa199321c6fa8e22e70","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"safedep/pmg","sha":"5ac0f275b83d9d5a9342c6aae977ec32fa330daa","version":"v1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1413,6 +1413,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - push_repo_memory - safe_outputs @@ -1482,6 +1484,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1580,6 +1590,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "dataflow-pr-discussion-dataset" with: @@ -1661,6 +1672,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1957,6 +1969,301 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "DataFlow PR & Discussion Dataset Builder" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/dataflow-pr-discussion-dataset.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: dataflowprdiscussiondataset-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"caveman_mode_goal_met","question":"Does the agent output show that the objective for experiment caveman_mode was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: small + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"caveman_mode_goal_met","question":"Does the agent output show that the objective for experiment caveman_mode was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "DataFlow PR & Discussion Dataset Builder" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/dataflow-pr-discussion-dataset.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/dataflowprdiscussiondataset + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/dataflow-pr-discussion-dataset.md b/.github/workflows/dataflow-pr-discussion-dataset.md index 141a80cef8f..c02ec770b68 100644 --- a/.github/workflows/dataflow-pr-discussion-dataset.md +++ b/.github/workflows/dataflow-pr-discussion-dataset.md @@ -106,6 +106,10 @@ experiments: start_date: "2026-06-05" issue: 37102 strict: true +evals: + - id: caveman_mode_goal_met + question: Does the agent output show that the objective for experiment caveman_mode was successfully completed? + --- {{#if experiments.caveman_mode == 'yes' }} diff --git a/.github/workflows/deep-report.lock.yml b/.github/workflows/deep-report.lock.yml index 4c2ae390e18..d37b0c0c948 100644 --- a/.github/workflows/deep-report.lock.yml +++ b/.github/workflows/deep-report.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"90019d972b4a10ef21432d059a2825400718df15a09ba5ab00824197248f7cd5","body_hash":"4b27cc83ecf8d8f35e71f85f3e07bd877f05a4b86e2b780b3f0f6d2438427a89","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"987ff54ce00966ea8bf9c81492b483d3abc913676a0d03d03210cf7d86921d8f","body_hash":"4b27cc83ecf8d8f35e71f85f3e07bd877f05a4b86e2b780b3f0f6d2438427a89","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1584,6 +1584,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - push_repo_memory - safe_outputs @@ -1653,6 +1655,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1751,6 +1761,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "deep-report" with: @@ -1831,6 +1842,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -2130,6 +2142,306 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Deep Report" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/deep-report.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.220" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: deepreport-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"output_format_goal_met","question":"Does the agent output show that the objective for experiment output_format was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.220 + - name: Execute Claude Code CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Allowed tools (sorted): + # - Bash + # - BashOutput + # - Edit(/tmp/*) + # - ExitPlanMode + # - Glob + # - Grep + # - KillBash + # - LS + # - MultiEdit(/tmp/*) + # - NotebookRead + # - Read + # - Read(/tmp/*) + # - Task + # - TodoWrite + # - Write(/tmp/*) + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.githubusercontent.com\",\"anthropic.com\",\"api.anthropic.com\",\"api.github.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"cdn.playwright.dev\",\"codeload.github.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"files.pythonhosted.org\",\"ghcr.io\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"playwright.download.prss.microsoft.com\",\"ppa.launchpad.net\",\"pypi.org\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"sentry.io\",\"statsig.anthropic.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --tty --env-all --exclude-env ANTHROPIC_API_KEY --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_MAX_RETRIES: 0 + ANTHROPIC_MODEL: small + BASH_DEFAULT_TIMEOUT_MS: 60000 + BASH_MAX_TIMEOUT_MS: 60000 + CLAUDE_CODE_DISABLE_FAST_MODE: 1 + DISABLE_BUG_COMMAND: 1 + DISABLE_ERROR_REPORTING: 1 + DISABLE_TELEMETRY: 1 + GH_AW_LLM_PROVIDER: anthropic + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + MCP_TIMEOUT: 120000 + MCP_TOOL_TIMEOUT: 60000 + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"output_format_goal_met","question":"Does the agent output show that the objective for experiment output_format was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Deep Report" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/deep-report.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.220" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/deepreport + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/deep-report.md b/.github/workflows/deep-report.md index 39513aa32de..89f0c66dd00 100644 --- a/.github/workflows/deep-report.md +++ b/.github/workflows/deep-report.md @@ -86,6 +86,10 @@ imports: - shared/otlp.md +evals: + - id: output_format_goal_met + question: Does the agent output show that the objective for experiment output_format was successfully completed? + --- ### DeepReport - Intelligence Gathering Agent diff --git a/.github/workflows/gpclean.lock.yml b/.github/workflows/gpclean.lock.yml index 4177c46a909..56661335083 100644 --- a/.github/workflows/gpclean.lock.yml +++ b/.github/workflows/gpclean.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e4c3a5bf0557117305ab95a2efa0de54b506c7cb3a4a209b4a17c7afa7a35e45","body_hash":"8dacbb099868174a8a7ab4550c4de15fb51fb46901f6bdb0c79abe26142fb01a","agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"b46a780c63f8c0a0bd71ab75d4317aeeea4ed0ebd569d75430c0eecb745e9199","body_hash":"8dacbb099868174a8a7ab4550c4de15fb51fb46901f6bdb0c79abe26142fb01a","agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1122,6 +1122,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs - update_cache_memory @@ -1189,6 +1191,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1286,6 +1296,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "gpclean" with: @@ -1363,6 +1374,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1652,6 +1664,301 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "GPL Dependency Cleaner (gpclean)" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/gpclean.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: gpclean-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"tool_verbosity_goal_met","question":"Does the agent output show that the objective for experiment tool_verbosity was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: small + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"tool_verbosity_goal_met","question":"Does the agent output show that the objective for experiment tool_verbosity was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "GPL Dependency Cleaner (gpclean)" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/gpclean.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/gpclean + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/gpclean.md b/.github/workflows/gpclean.md index 501c4751726..1462eee6f74 100644 --- a/.github/workflows/gpclean.md +++ b/.github/workflows/gpclean.md @@ -89,6 +89,10 @@ steps: echo "📊 SBOM contains ${PACKAGE_COUNT} packages" fi +evals: + - id: tool_verbosity_goal_met + question: Does the agent output show that the objective for experiment tool_verbosity was successfully completed? + --- {{#if experiments.tool_verbosity == 'minimal_toolset' }} diff --git a/.github/workflows/smoke-antigravity.lock.yml b/.github/workflows/smoke-antigravity.lock.yml index 91a8b3c06df..026d07460d6 100644 --- a/.github/workflows/smoke-antigravity.lock.yml +++ b/.github/workflows/smoke-antigravity.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"3a92aeba8f210faa9e14c51c06db2bbe2d7a61e905f7b286873e6e9d14565820","body_hash":"d5abdd42decd63646c50fbac99cf05a2fbc3cb20fb3388a952814d67ab014a86","strict":true,"agent_id":"antigravity","engine_versions":{"antigravity":"1.0.2-6113393518706688"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"87242582a22a94f93ef85ee2623dc60bfc4e84e0360ac7a8e97678036e6086f8","body_hash":"d5abdd42decd63646c50fbac99cf05a2fbc3cb20fb3388a952814d67ab014a86","strict":true,"agent_id":"antigravity","engine_versions":{"antigravity":"1.0.2-6113393518706688"}} # gh-aw-manifest: {"version":1,"secrets":["ANTIGRAVITY_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}],"has_pull_request":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1198,6 +1198,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs - update_cache_memory @@ -1264,6 +1266,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1361,6 +1371,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-antigravity" with: @@ -1437,6 +1448,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_ENGINE_API_HOSTS: "generativelanguage.googleapis.com" GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} @@ -1732,6 +1744,214 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Antigravity" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-antigravity.lock.yml@${{ github.ref }} + GH_AW_INFO_ENGINE_ID: "antigravity" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smokeantigravity-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Install Antigravity CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_antigravity_cli.sh" "${ENGINE_VERSION}" + env: + ENGINE_VERSION: 1.0.2-6113393518706688 + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Write Antigravity Config + if: always() + continue-on-error: true + run: | + mkdir -p "$GITHUB_WORKSPACE/.antigravity" + SETTINGS="$GITHUB_WORKSPACE/.antigravity/settings.json" + BASE_CONFIG="$GH_AW_ANTIGRAVITY_BASE_CONFIG" + if [ -f "$SETTINGS" ]; then + MERGED=$(jq -n --argjson base "$BASE_CONFIG" --argjson existing "$(cat "$SETTINGS")" '$existing * $base') + echo "$MERGED" > "$SETTINGS" + else + echo "$BASE_CONFIG" > "$SETTINGS" + fi + env: + GH_AW_ANTIGRAVITY_BASE_CONFIG: '{"context":{"includeDirectories":["/tmp/"]},"tools":{"core":["glob","grep_search","list_directory","read_file","read_many_files","run_shell_command"]}}' + - name: Execute Antigravity CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.googleapis.com\",\"generativelanguage.googleapis.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"targets\":{\"gemini\":{\"host\":\"generativelanguage.googleapis.com\"}},\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env ANTIGRAVITY_API_KEY --exclude-env GEMINI_API_KEY --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && cd "${GITHUB_WORKSPACE}" && agy --dangerously-skip-permissions --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + ANTIGRAVITY_API_BASE_URL: http://host.docker.internal:10003 + ANTIGRAVITY_API_KEY: ${{ secrets.ANTIGRAVITY_API_KEY }} + ANTIGRAVITY_CLI_TRUST_WORKSPACE: true + ANTIGRAVITY_MODEL: small + DEBUG: antigravity-cli:* + GEMINI_API_KEY: ${{ secrets.ANTIGRAVITY_API_KEY }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: > (github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id) && @@ -1790,6 +2010,83 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Antigravity" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-antigravity.lock.yml@${{ github.ref }} + GH_AW_INFO_ENGINE_ID: "antigravity" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smokeantigravity + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-antigravity.md b/.github/workflows/smoke-antigravity.md index 862e84f66ee..6ca61f9f607 100644 --- a/.github/workflows/smoke-antigravity.md +++ b/.github/workflows/smoke-antigravity.md @@ -72,6 +72,10 @@ safe-outputs: timeout-minutes: 10 features: gh-aw-detection: false +evals: + - id: sub_agent_strategy_goal_met + question: Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed? + --- # Smoke Test: Antigravity Engine Validation diff --git a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml index da8462c57ca..135cb2b21bb 100644 --- a/.github/workflows/smoke-copilot-aoai-apikey.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-apikey.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"37080dd64465f506c15b5a6782f7d8829a80d536fc08438808f18c616108184f","body_hash":"35025aaffbb70f75f0abcee6e5039fcc3f5c5d3dcb8931bba69477605f6c0df0","strict":true,"agent_id":"copilot","agent_model":"o4-mini-aw","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a4b1e991228167a977c0e1936b2433afe9c99a671a480b1b1b32155a7a70a63b","body_hash":"35025aaffbb70f75f0abcee6e5039fcc3f5c5d3dcb8931bba69477605f6c0df0","strict":true,"agent_id":"copilot","agent_model":"o4-mini-aw","engine_versions":{"copilot":"1.0.75"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","FOUNDRY_API_KEY","FOUNDRY_OPENAI_ENDPOINT","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"},{"image":"ghcr.io/github/serena-mcp-server:latest","digest":"sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5","pinned_image":"ghcr.io/github/serena-mcp-server:latest@sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -2240,6 +2240,8 @@ jobs: - agent - check_token_telemetry - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs - send_slack_message @@ -2312,6 +2314,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -2409,6 +2419,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-copilot-aoai-apikey" with: @@ -2484,6 +2495,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -2800,6 +2812,224 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Copilot - AOAI (apikey)" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-copilot-aoai-apikey.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smokecopilotaoaiapikey-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"caveman_goal_met","question":"Does the agent output show that the objective for experiment caveman was successfully completed?"},{"id":"subagent_model_goal_met","question":"Does the agent output show that the objective for experiment subagent_model was successfully completed?"}]' + GH_AW_EVALS_MODEL: "o4-mini-aw" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.grafana.net\",\"*.sentry.io\",\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_PROVIDER_API_KEY --exclude-env COPILOT_PROVIDER_BASE_URL --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --autopilot --max-autopilot-continues 2 --allow-all-tools --no-custom-instructions --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_MODEL: o4-mini-aw + COPILOT_PROVIDER_API_KEY: ${{ secrets.FOUNDRY_API_KEY }} + COPILOT_PROVIDER_BASE_URL: ${{ secrets.FOUNDRY_OPENAI_ENDPOINT }} + COPILOT_PROVIDER_WIRE_API: responses + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"caveman_goal_met","question":"Does the agent output show that the objective for experiment caveman was successfully completed?"},{"id":"subagent_model_goal_met","question":"Does the agent output show that the objective for experiment subagent_model was successfully completed?"}]' + GH_AW_EVALS_MODEL: "o4-mini-aw" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: > (fromJSON(github.event.inputs.aw_context || '{}').event_type == 'pull_request') && (fromJSON(github.event.inputs.aw_context || @@ -2860,6 +3090,85 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Copilot - AOAI (apikey)" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-copilot-aoai-apikey.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smokecopilotaoaiapikey + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-copilot-aoai-apikey.md b/.github/workflows/smoke-copilot-aoai-apikey.md index a0c3cb97ee0..e10543fcbb2 100644 --- a/.github/workflows/smoke-copilot-aoai-apikey.md +++ b/.github/workflows/smoke-copilot-aoai-apikey.md @@ -148,6 +148,12 @@ experiments: subagent_model: [small, large] features: gh-aw-detection: false +evals: + - id: caveman_goal_met + question: Does the agent output show that the objective for experiment caveman was successfully completed? + - id: subagent_model_goal_met + question: Does the agent output show that the objective for experiment subagent_model was successfully completed? + --- # Smoke Test: Copilot Engine Validation (AOAI apikey BYOK) diff --git a/.github/workflows/smoke-copilot-aoai-entra.lock.yml b/.github/workflows/smoke-copilot-aoai-entra.lock.yml index 31e0e1d492c..069ecf349df 100644 --- a/.github/workflows/smoke-copilot-aoai-entra.lock.yml +++ b/.github/workflows/smoke-copilot-aoai-entra.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"eacd44481d0c753604cddcf4efed75c2ad331fda3401981f38e7e40f105c152f","body_hash":"e922bb9add6d1e143a4a2af49416ffb02e122afc214785d283442d1365100c26","strict":true,"agent_id":"copilot","agent_model":"o4-mini-aw","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"0a7e20122298fdc5fcbbde4eca4f3555ee923794bf9a0d6097e3248b017b9845","body_hash":"e922bb9add6d1e143a4a2af49416ffb02e122afc214785d283442d1365100c26","strict":true,"agent_id":"copilot","agent_model":"o4-mini-aw","engine_versions":{"copilot":"1.0.75"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","FOUNDRY_OPENAI_ENDPOINT","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-go","sha":"b7ad1dad31e06c5925ef5d2fc7ad053ef454303e","version":"v7.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"docker/build-push-action","sha":"53b7df96c91f9c12dcc8a07bcb9ccacbed38856a","version":"v7.3.0"},{"repo":"docker/setup-buildx-action","sha":"bb05f3f5519dd87d3ba754cc423b652a5edd6d2c","version":"v4.2.0"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"},{"image":"ghcr.io/github/serena-mcp-server:latest","digest":"sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5","pinned_image":"ghcr.io/github/serena-mcp-server:latest@sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -2259,6 +2259,8 @@ jobs: - agent - check_token_telemetry - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs - send_slack_message @@ -2332,6 +2334,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -2429,6 +2439,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-copilot-aoai-entra" with: @@ -2504,6 +2515,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -2825,6 +2837,229 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + environment: aoai-model + permissions: + contents: read + id-token: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Copilot - AOAI (Entra)" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-copilot-aoai-entra.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smokecopilotaoaientra-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"caveman_goal_met","question":"Does the agent output show that the objective for experiment caveman was successfully completed?"},{"id":"subagent_model_goal_met","question":"Does the agent output show that the objective for experiment subagent_model was successfully completed?"}]' + GH_AW_EVALS_MODEL: "o4-mini-aw" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.grafana.net\",\"*.sentry.io\",\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"login.microsoftonline.com\",\"management.azure.com\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_PROVIDER_BASE_URL --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --autopilot --max-autopilot-continues 2 --allow-all-tools --no-custom-instructions --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_AUTH_AZURE_CLIENT_ID: adb907fd-188c-4029-b67f-2559d96b2f1b + AWF_AUTH_AZURE_TENANT_ID: 398a6654-997b-47e9-b12b-9515b896b4de + AWF_AUTH_PROVIDER: azure + AWF_AUTH_TYPE: github-oidc + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_MODEL: o4-mini-aw + COPILOT_PROVIDER_BASE_URL: ${{ secrets.FOUNDRY_OPENAI_ENDPOINT }} + COPILOT_PROVIDER_WIRE_API: responses + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"caveman_goal_met","question":"Does the agent output show that the objective for experiment caveman was successfully completed?"},{"id":"subagent_model_goal_met","question":"Does the agent output show that the objective for experiment subagent_model was successfully completed?"}]' + GH_AW_EVALS_MODEL: "o4-mini-aw" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: > (fromJSON(github.event.inputs.aw_context || '{}').event_type == 'pull_request') && (fromJSON(github.event.inputs.aw_context || @@ -2886,6 +3121,85 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Copilot - AOAI (Entra)" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-copilot-aoai-entra.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smokecopilotaoaientra + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-copilot-aoai-entra.md b/.github/workflows/smoke-copilot-aoai-entra.md index 9cee0a97532..e6a43585913 100644 --- a/.github/workflows/smoke-copilot-aoai-entra.md +++ b/.github/workflows/smoke-copilot-aoai-entra.md @@ -158,6 +158,12 @@ experiments: subagent_model: [small, large] features: gh-aw-detection: false +evals: + - id: caveman_goal_met + question: Does the agent output show that the objective for experiment caveman was successfully completed? + - id: subagent_model_goal_met + question: Does the agent output show that the objective for experiment subagent_model was successfully completed? + --- # Smoke Test: Copilot Engine Validation (AOAI Entra BYOK) diff --git a/.github/workflows/smoke-copilot-sub-agents.lock.yml b/.github/workflows/smoke-copilot-sub-agents.lock.yml index bd8545d3f59..0bebcd1a6dd 100644 --- a/.github/workflows/smoke-copilot-sub-agents.lock.yml +++ b/.github/workflows/smoke-copilot-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c2df8673562c406db822916fc40e112da4f433e431e57d81a15e5b173899fbdc","body_hash":"66dd32a20bb7d4a839aca20b4c9729240d2574e69a5f13f3d4aaa484472974a4","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"421af724ab2acd2be54cba4e75bda77d5ae8dea6dbf371e48b8e828416c88cbd","body_hash":"66dd32a20bb7d4a839aca20b4c9729240d2574e69a5f13f3d4aaa484472974a4","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.75","copilot-sdk":"1.0.8"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1034,6 +1034,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs if: > @@ -1100,6 +1102,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1197,6 +1207,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-copilot-sub-agents" with: @@ -1272,6 +1283,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1560,6 +1572,309 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + copilot-requests: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Copilot Sub Agents" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-copilot-sub-agents.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smokecopilotsubagents-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "gpt-5.3-codex" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Install GitHub Copilot SDK (Node.js) + run: cd "${GITHUB_WORKSPACE}" && npm install --ignore-scripts --no-save @github/copilot-sdk@1.0.8 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_WORKSPACE_NODE_MODULES="${GITHUB_WORKSPACE:-$PWD}/node_modules"; if [ -d "$GH_AW_WORKSPACE_NODE_MODULES" ]; then export NODE_PATH="${GH_AW_WORKSPACE_NODE_MODULES}${NODE_PATH:+:${NODE_PATH}}"; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs "$GH_AW_NODE_EXEC" "${RUNNER_TEMP}/gh-aw/actions/copilot_sdk_driver.cjs" /usr/local/bin/copilot' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ github.token }} + COPILOT_MODEL: gpt-5.3-codex + COPILOT_SDK_URI: http://127.0.0.1:3002 + GH_AW_COPILOT_SDK_DRIVER: 1 + GH_AW_COPILOT_SDK_SERVER_ARGS: '["--headless","--no-auto-update","--port","3002","--add-dir","/tmp/gh-aw/","--log-level","all","--log-dir","/tmp/gh-aw/sandbox/agent/logs/","--disable-builtin-mcps","--no-ask-user","--allow-all-tools","--no-custom-instructions"]' + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TOOL_DENIALS: 3 + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + S2STOKENS: true + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "gpt-5.3-codex" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Copilot Sub Agents" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-copilot-sub-agents.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smokecopilotsubagents + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-copilot-sub-agents.md b/.github/workflows/smoke-copilot-sub-agents.md index e0d54fbb03d..0a6196f86cc 100644 --- a/.github/workflows/smoke-copilot-sub-agents.md +++ b/.github/workflows/smoke-copilot-sub-agents.md @@ -46,6 +46,10 @@ safe-outputs: timeout-minutes: 10 features: gh-aw-detection: false +evals: + - id: sub_agent_strategy_goal_met + question: Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed? + --- # Smoke Test: Copilot SDK Inline Sub-Agents diff --git a/.github/workflows/smoke-gemini.lock.yml b/.github/workflows/smoke-gemini.lock.yml index e2a524b820c..3160d3292c9 100644 --- a/.github/workflows/smoke-gemini.lock.yml +++ b/.github/workflows/smoke-gemini.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"a43b274adbcfa67b48edc1e6c8bc75d8b0891de537ba8d3d2c9ef67ca6378191","body_hash":"a3b4d09c6c432e1207b25d2087f674156c49ee59fc3ff69ca38cabf82e96a840","strict":true,"agent_id":"gemini","engine_versions":{"gemini":"0.39.1"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"e231a7b20107ae599f701463d00b941cd0f9c51fdd3cffddddbeeddea89dcab8","body_hash":"a3b4d09c6c432e1207b25d2087f674156c49ee59fc3ff69ca38cabf82e96a840","strict":true,"agent_id":"gemini","engine_versions":{"gemini":"0.39.1"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GEMINI_API_KEY","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}],"has_pull_request":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1272,6 +1272,8 @@ jobs: - agent - check_token_telemetry - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs - update_cache_memory @@ -1339,6 +1341,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1436,6 +1446,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-gemini" with: @@ -1512,6 +1523,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_ENGINE_API_HOSTS: "generativelanguage.googleapis.com" GH_AW_LOCKDOWN_CHECK_FAILED: ${{ needs.activation.outputs.lockdown_check_failed }} @@ -1810,6 +1822,217 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Gemini" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-gemini.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "0.39.1" + GH_AW_INFO_ENGINE_ID: "gemini" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smokegemini-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Install Gemini CLI + run: npm install --ignore-scripts -g @google/gemini-cli@0.39.1 + - name: Write Gemini Config + if: always() + continue-on-error: true + run: | + mkdir -p "$GITHUB_WORKSPACE/.gemini" + SETTINGS="$GITHUB_WORKSPACE/.gemini/settings.json" + BASE_CONFIG="$GH_AW_GEMINI_BASE_CONFIG" + if [ -f "$SETTINGS" ]; then + MERGED=$(jq -n --argjson base "$BASE_CONFIG" --argjson existing "$(cat "$SETTINGS")" '$existing * $base') + echo "$MERGED" > "$SETTINGS" + else + echo "$BASE_CONFIG" > "$SETTINGS" + fi + env: + GH_AW_GEMINI_BASE_CONFIG: '{"context":{"includeDirectories":["/tmp/"]},"tools":{"core":["glob","grep_search","list_directory","read_file","read_many_files","run_shell_command"]}}' + - name: Execute Gemini CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.googleapis.com\",\"generativelanguage.googleapis.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"targets\":{\"gemini\":{\"host\":\"generativelanguage.googleapis.com\"}},\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env GEMINI_API_KEY --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && cd "${GITHUB_WORKSPACE}" && gemini --yolo --skip-trust --output-format stream-json --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + DEBUG: gemini-cli:* + GEMINI_API_BASE_URL: http://host.docker.internal:10003 + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GEMINI_CLI_TRUST_WORKSPACE: true + GEMINI_MODEL: small + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: > (github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id) && @@ -1869,6 +2092,84 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Gemini" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-gemini.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "0.39.1" + GH_AW_INFO_ENGINE_ID: "gemini" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smokegemini + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-gemini.md b/.github/workflows/smoke-gemini.md index 2c99c11994a..d6b29dba799 100644 --- a/.github/workflows/smoke-gemini.md +++ b/.github/workflows/smoke-gemini.md @@ -71,6 +71,10 @@ safe-outputs: run-success: "🚀 [{workflow_name}]({run_url}) **MISSION COMPLETE!** Gemini has spoken. ✨" run-failure: "⚠️ [{workflow_name}]({run_url}) {status}. Gemini encountered unexpected challenges..." timeout-minutes: 10 +evals: + - id: sub_agent_strategy_goal_met + question: Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed? + --- # Smoke Test: Gemini Engine Validation diff --git a/.github/workflows/smoke-project.lock.yml b/.github/workflows/smoke-project.lock.yml index 376e56d34bb..91c939845e7 100644 --- a/.github/workflows/smoke-project.lock.yml +++ b/.github/workflows/smoke-project.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"168a5e6e5b002f538cfb700c71be4eb15088d2a56ac50ac6803169c503aad013","body_hash":"2652d8b9299d8344d5bb0a6a79d73c81dd8f9d870d6afe19e12eba02073ed63f","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"5e2b9f59c727829b8b3ae90bf9456036b560fd17188dba88b2c8ce9ac4554503","body_hash":"2652d8b9299d8344d5bb0a6a79d73c81dd8f9d870d6afe19e12eba02073ed63f","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GH_AW_PROJECT_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}],"has_pull_request":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1389,6 +1389,8 @@ jobs: - agent - check_token_telemetry - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs if: > @@ -1456,6 +1458,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1553,6 +1563,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-project" with: @@ -1630,6 +1641,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1940,6 +1952,222 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Project" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-project.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smokeproject-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"prompt_style_test_goal_met","question":"Does the agent output show that the objective for experiment prompt_style_test was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: small + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"prompt_style_test_goal_met","question":"Does the agent output show that the objective for experiment prompt_style_test was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: > (github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id) && @@ -2000,6 +2228,85 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Project" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-project.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smokeproject + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-project.md b/.github/workflows/smoke-project.md index 0ab241adb6f..9d0d3073f07 100644 --- a/.github/workflows/smoke-project.md +++ b/.github/workflows/smoke-project.md @@ -92,6 +92,10 @@ imports: - shared/token-telemetry-check.md features: gh-aw-detection: false +evals: + - id: prompt_style_test_goal_met + question: Does the agent output show that the objective for experiment prompt_style_test was successfully completed? + --- # Smoke Test: Project Operations Validation diff --git a/.github/workflows/smoke-temporary-id.lock.yml b/.github/workflows/smoke-temporary-id.lock.yml index 0269260310d..4411bae3dad 100644 --- a/.github/workflows/smoke-temporary-id.lock.yml +++ b/.github/workflows/smoke-temporary-id.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8edff1954fd913b37dccc602d01c690bee7052e688320d85d3f36abb70e551ab","body_hash":"2d114653af3d3ce42c82d1254ec15f52a176f4484f8352cfdb2be6593193faf5","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"8ebccce58be7cb29a99eb12cff780ee69741cf719485bceb4cf22005da990ea7","body_hash":"2d114653af3d3ce42c82d1254ec15f52a176f4484f8352cfdb2be6593193faf5","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.75"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}],"has_pull_request":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1240,6 +1240,8 @@ jobs: - agent - check_token_telemetry - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs if: > @@ -1307,6 +1309,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1404,6 +1414,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "smoke-temporary-id" with: @@ -1481,6 +1492,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1789,6 +1801,222 @@ jobs: } } + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Temporary ID" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-temporary-id.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: smoketemporaryid-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: small + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"sub_agent_strategy_goal_met","question":"Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: > (github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id) && @@ -1849,6 +2077,85 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Smoke Temporary ID" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/smoke-temporary-id.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/smoketemporaryid + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/smoke-temporary-id.md b/.github/workflows/smoke-temporary-id.md index a0b26b77329..6caa9e3dac7 100644 --- a/.github/workflows/smoke-temporary-id.md +++ b/.github/workflows/smoke-temporary-id.md @@ -70,6 +70,10 @@ experiments: tags: [cost-efficiency, sub-agents, smoke-tests] features: gh-aw-detection: false +evals: + - id: sub_agent_strategy_goal_met + question: Does the agent output show that the objective for experiment sub_agent_strategy was successfully completed? + --- # Smoke Test: Temporary ID Functionality diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index 6792e465c85..787fd36ccfe 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1,5 +1,5 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"63fef4db9b9592dc924de548310c91021b3431ce5ea43c2d1a4483a3f9b3e960","body_hash":"55f6950223bf1ed2f2053b4b482a63ca2d659f29f6d5ca02d1e03f1033d5a68e","strict":true,"agent_id":"copilot","agent_model":"${{ needs.activation.outputs.model_size }}","engine_versions":{"copilot":"1.0.75"}} -# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}],"has_pull_request":true} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"ebcf85b7703412c989380d57848ec73339f49efce7c6c69b831437c2f31931ed","body_hash":"55f6950223bf1ed2f2053b4b482a63ca2d659f29f6d5ca02d1e03f1033d5a68e","strict":true,"agent_id":"copilot","agent_model":"${{ needs.activation.outputs.model_size }}","engine_versions":{"copilot":"1.0.75"}} +# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"}],"has_pull_request":true} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # # ___ _ _ @@ -48,6 +48,7 @@ # - actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 # - actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 (source v9) +# - actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 # - actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 # # Container images used: @@ -1183,6 +1184,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs if: > @@ -1250,6 +1253,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1347,6 +1358,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "test-quality-sentinel" with: @@ -1422,6 +1434,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1701,6 +1714,227 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + copilot-requests: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Test Quality Sentinel" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/test-quality-sentinel.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: testqualitysentinel-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"model_size_goal_met","question":"Does the agent output show that the objective for experiment model_size was successfully completed?"}]' + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_EVALS_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-5' }} + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install GitHub Copilot CLI + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_copilot_cli.sh" 1.0.75 + env: + GH_HOST: github.com + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Execute GitHub Copilot CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + trap 'gh_aw_exit_code=$?; mkdir -p /tmp/gh-aw >/dev/null 2>&1 || true; printf "%s" "$gh_aw_exit_code" > /tmp/gh-aw/agent_execution_exit_code.txt || true; rm -f "$HOME/.copilot/settings.json"' EXIT + mkdir -p "$HOME/.copilot" + printf '%s' '{"builtInAgents":{"rubberDuck":false}}' > "$HOME/.copilot/settings.json" + export XDG_CONFIG_HOME="$HOME" + touch /tmp/gh-aw/agent-step-summary.md + GH_AW_NODE_BIN=$(command -v node 2>/dev/null || true) + export GH_AW_NODE_BIN + export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${GH_AW_MAX_AI_CREDITS:-400}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"api.business.githubcopilot.com\",\"api.enterprise.githubcopilot.com\",\"api.github.com\",\"api.githubcopilot.com\",\"api.individual.githubcopilot.com\",\"github.com\",\"host.docker.internal\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"telemetry.enterprise.githubcopilot.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --env-all --exclude-env COPILOT_GITHUB_TOKEN --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/copilot_harness.cjs /usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --disable-builtin-mcps --no-ask-user --autopilot --max-autopilot-continues 15 --allow-all-tools --add-dir "${GITHUB_WORKSPACE}" --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + AWF_REFLECT_ENABLED: 1 + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode + COPILOT_GITHUB_TOKEN: ${{ github.token }} + COPILOT_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_LLM_PROVIDER: github + GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }} + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_EVALS_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-5' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_TIMEOUT_MINUTES: 20 + GH_AW_VERSION: dev + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_AW: true + GITHUB_COPILOT_INTEGRATION_ID: agentic-workflows + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + RUNNER_TEMP: ${{ runner.temp }} + S2STOKENS: true + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"model_size_goal_met","question":"Does the agent output show that the objective for experiment model_size was successfully completed?"}]' + GH_AW_EVALS_MODEL: ${{ needs.activation.outputs.model_size }} + GH_AW_MODEL_FALLBACK: ${{ vars.GH_AW_MODEL_EVALS_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-5' }} + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + pre_activation: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.id == github.repository_id runs-on: ubuntu-slim @@ -1759,6 +1993,85 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Test Quality Sentinel" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/test-quality-sentinel.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.75" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/testqualitysentinel + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/test-quality-sentinel.md b/.github/workflows/test-quality-sentinel.md index 768666d5d73..bcb9167b690 100644 --- a/.github/workflows/test-quality-sentinel.md +++ b/.github/workflows/test-quality-sentinel.md @@ -195,6 +195,10 @@ experiments: min_samples: 70 weight: [50, 50] start_date: "2026-07-05" +evals: + - id: model_size_goal_met + question: Does the agent output show that the objective for experiment model_size was successfully completed? + --- # Test Quality Sentinel 🧪 diff --git a/.github/workflows/typist.lock.yml b/.github/workflows/typist.lock.yml index 52cbeb296fb..9b77b471d66 100644 --- a/.github/workflows/typist.lock.yml +++ b/.github/workflows/typist.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"f01ccb48dbe111dfdf865bd36f4c7c7599e7de354fdf71c6a70bfce6c25c0d8d","body_hash":"709f09df44ca139b9f329e5283ddfe139cf225cf4ea70718080dd2fc7f66d342","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"107e15e2e771d982ebde756d6d9a98e43f0a0746dbcb76cf11b8163a6207a787","body_hash":"709f09df44ca139b9f329e5283ddfe139cf225cf4ea70718080dd2fc7f66d342","strict":true,"agent_id":"claude","engine_versions":{"claude":"2.1.220"}} # gh-aw-manifest: {"version":1,"secrets":["ANTHROPIC_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42","digest":"sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42","digest":"sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42","digest":"sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f","pinned_image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.27.42@sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42","digest":"sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.6","digest":"sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.6@sha256:fecabec51bbc41f2ad61076d6bcd9a36ef23b142e672a444e054d37fc29de93c"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748","pinned_image":"ghcr.io/github/gh-aw-node@sha256:a8082161d7dceda14b68f32eb39d0eaa96b825d07f5895b096afab9d9e0c7748"},{"image":"ghcr.io/github/github-mcp-server:v1.7.0","digest":"sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308","pinned_image":"ghcr.io/github/github-mcp-server:v1.7.0@sha256:c491ffdf6f4c85cb5397021bc655edb8ab825c6f5f568e7597d77a1bd7c4d308"},{"image":"ghcr.io/github/serena-mcp-server:latest","digest":"sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5","pinned_image":"ghcr.io/github/serena-mcp-server:latest@sha256:bf343399e3725c45528f531a230f3a04521d4cdef29f9a5af6282ff0d3c393c5"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -1179,6 +1179,8 @@ jobs: - activation - agent - detection + - evals + - push_evals_state - push_experiments_state - safe_outputs if: > @@ -1246,6 +1248,14 @@ jobs: with: name: safe-outputs-items path: /tmp/gh-aw/ + - name: Download evals artifact + id: download-evals-artifact + if: always() + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: evals + path: /tmp/gh-aw/evals/ - name: Collect usage artifact files if: always() continue-on-error: true @@ -1343,6 +1353,7 @@ jobs: GH_AW_NOOP_REPORT_AS_ISSUE: "true" GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_AMBIENT_CONTEXT: ${{ needs.agent.outputs.ambient_context }} GH_AW_WORKFLOW_ID: "typist" with: @@ -1419,6 +1430,7 @@ jobs: GH_AW_UNKNOWN_MODEL_AI_CREDITS: ${{ needs.agent.outputs.unknown_model_ai_credits || 'false' }} GH_AW_AIC: ${{ needs.agent.outputs.aic }} GH_AW_THREAT_DETECTION_AIC: ${{ needs.detection.outputs.aic }} + GH_AW_EVALS_AIC: ${{ needs.evals.outputs.aic }} GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} GH_AW_MCP_POLICY_ERROR: ${{ needs.agent.outputs.mcp_policy_error }} @@ -1677,6 +1689,306 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/conclude_threat_detection.sh" /tmp/gh-aw/threat-detection/detection_result.json + evals: + needs: + - activation + - agent + - detection + if: always() && needs.agent.result != 'skipped' + runs-on: ubuntu-latest + permissions: + contents: read + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + outputs: + aic: ${{ steps.parse-mcp-gateway.outputs.aic }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Typist - Go Type Analysis" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/typist.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.220" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: agent + path: /tmp/gh-aw/ + - name: Setup agent output environment variable + id: setup-agent-output-env + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/ + find "/tmp/gh-aw/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: typist-experiment + path: /tmp/gh-aw/experiments/ + # --- BinEval Evaluations --- + - name: Clean stale firewall files from agent artifact + run: | + rm -rf /tmp/gh-aw/sandbox/firewall/logs + rm -rf /tmp/gh-aw/sandbox/firewall/audit + - name: Download container images + run: bash "${RUNNER_TEMP}/gh-aw/actions/download_docker_images.sh" ghcr.io/github/gh-aw-firewall/agent:0.27.42@sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b ghcr.io/github/gh-aw-firewall/api-proxy:0.27.42@sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607 ghcr.io/github/gh-aw-firewall/squid:0.27.42@sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0 + - name: Prepare evals files + run: | + mkdir -p /tmp/gh-aw/evals + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/evals/agent_output.json 2>/dev/null || true + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/evals/prompt.txt 2>/dev/null || true + ls -la /tmp/gh-aw/evals/ 2>/dev/null || true + - name: Setup BinEval evaluations + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"tone_style_goal_met","question":"Does the agent output show that the objective for experiment tone_style was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: setup + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Ensure evals directory and log + run: | + mkdir -p /tmp/gh-aw/evals + touch /tmp/gh-aw/evals/evals.log + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '24' + package-manager-cache: false + - name: Install AWF binary + run: bash "${RUNNER_TEMP}/gh-aw/actions/install_awf_binary.sh" v0.27.42 + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code@2.1.220 + - name: Execute Claude Code CLI + if: always() + continue-on-error: true + id: evals_agentic_execution + # Allowed tools (sorted): + # - Bash + # - BashOutput + # - Edit(/tmp/*) + # - ExitPlanMode + # - Glob + # - Grep + # - KillBash + # - LS + # - MultiEdit(/tmp/*) + # - NotebookRead + # - Read + # - Read(/tmp/*) + # - Task + # - TodoWrite + # - Write(/tmp/*) + timeout-minutes: 20 + run: | + set -o pipefail + printf '%s' "$(date +%s%3N)" > /tmp/gh-aw/agent_cli_start_ms.txt + touch /tmp/gh-aw/agent-step-summary.md + (umask 177 && touch /tmp/gh-aw/evals/evals.log) + GH_AW_MAX_AI_CREDITS="${{ vars.GH_AW_DEFAULT_EVALS_MAX_AI_CREDITS || '400' }}" + printf '%s\n' "{\"\$schema\":\"https://github.com/github/gh-aw-firewall/releases/download/v0.27.42/awf-config.schema.json\",\"network\":{\"allowDomains\":[\"*.githubusercontent.com\",\"anthropic.com\",\"api.anthropic.com\",\"api.github.com\",\"api.snapcraft.io\",\"archive.ubuntu.com\",\"azure.archive.ubuntu.com\",\"cdn.playwright.dev\",\"codeload.github.com\",\"crl.geotrust.com\",\"crl.globalsign.com\",\"crl.identrust.com\",\"crl.sectigo.com\",\"crl.thawte.com\",\"crl.usertrust.com\",\"crl.verisign.com\",\"crl3.digicert.com\",\"crl4.digicert.com\",\"crls.ssl.com\",\"files.pythonhosted.org\",\"ghcr.io\",\"github-cloud.githubusercontent.com\",\"github-cloud.s3.amazonaws.com\",\"github.com\",\"host.docker.internal\",\"json-schema.org\",\"json.schemastore.org\",\"keyserver.ubuntu.com\",\"lfs.github.com\",\"objects.githubusercontent.com\",\"ocsp.digicert.com\",\"ocsp.geotrust.com\",\"ocsp.globalsign.com\",\"ocsp.identrust.com\",\"ocsp.sectigo.com\",\"ocsp.ssl.com\",\"ocsp.thawte.com\",\"ocsp.usertrust.com\",\"ocsp.verisign.com\",\"packagecloud.io\",\"packages.cloud.google.com\",\"packages.microsoft.com\",\"playwright.download.prss.microsoft.com\",\"ppa.launchpad.net\",\"pypi.org\",\"raw.githubusercontent.com\",\"registry.npmjs.org\",\"s.symcb.com\",\"s.symcd.com\",\"security.ubuntu.com\",\"sentry.io\",\"statsig.anthropic.com\",\"ts-crl.ws.symantec.com\",\"ts-ocsp.ws.symantec.com\"]},\"apiProxy\":{\"enabled\":true,\"enableTokenSteering\":true,\"maxRuns\":500,\"maxAiCredits\":${GH_AW_MAX_AI_CREDITS},\"maxCacheMisses\":5,\"models\":{\"agent\":[\"sonnet-6x\",\"gpt-5.4\",\"gpt-5.5\",\"gpt-5.6\",\"gpt-5.3\",\"gemini-pro\",\"any\"],\"antigravity\":[\"copilot/antigravity*\",\"google/antigravity*\",\"gemini/antigravity*\"],\"any\":[\"copilot/*\",\"anthropic/*\",\"openai/*\",\"google/*\",\"gemini/*\"],\"auto\":[\"copilot/auto\",\"large\"],\"claude\":[\"agent\"],\"codex\":[\"agent\"],\"coding\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\",\"gpt-5-codex\",\"kimi\"],\"computer-use\":[\"copilot/*computer-use*\",\"google/*computer-use*\",\"gemini/*computer-use*\",\"openai/*computer-use*\"],\"copilot\":[\"agent\"],\"deep-research\":[\"copilot/deep-research*\",\"copilot/o3-deep-research*\",\"copilot/o4-mini-deep-research*\",\"google/deep-research*\",\"gemini/deep-research*\",\"openai/o3-deep-research*\",\"openai/o4-mini-deep-research*\"],\"fable\":[\"copilot/*fable*\",\"anthropic/*fable*\"],\"gemini\":[\"agent\"],\"gemini-3-flash\":[\"copilot/gemini-3*flash*\",\"google/gemini-3*flash*\",\"gemini/gemini-3*flash*\"],\"gemini-3-pro\":[\"copilot/gemini-3*pro*\",\"google/gemini-3*pro*\",\"google/nano-banana*\",\"gemini/gemini-3*pro*\"],\"gemini-3.1-flash\":[\"copilot/gemini-3.1*flash*\",\"google/gemini-3.1*flash*\",\"gemini/gemini-3.1*flash*\"],\"gemini-3.1-pro\":[\"copilot/gemini-3.1*pro*\",\"google/gemini-3.1*pro*\",\"gemini/gemini-3.1*pro*\"],\"gemini-3.5-flash\":[\"copilot/gemini-3.5*flash*\",\"google/gemini-3.5*flash*\",\"gemini/gemini-3.5*flash*\"],\"gemini-3.6-flash\":[\"copilot/gemini-3.6*flash*\",\"google/gemini-3.6*flash*\",\"gemini/gemini-3.6*flash*\"],\"gemini-flash\":[\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"],\"gemini-flash-lite\":[\"copilot/gemini-*flash*lite*\",\"google/gemini-*flash*lite*\",\"gemini/gemini-*flash*lite*\"],\"gemini-omni\":[\"copilot/gemini-omni*\",\"google/gemini-omni*\",\"gemini/gemini-omni*\"],\"gemini-pro\":[\"copilot/gemini-*pro*\",\"google/gemini-*pro*\",\"gemini/gemini-*pro*\"],\"gemma\":[\"copilot/gemma*\",\"google/gemma*\",\"gemini/gemma*\"],\"gpt-5\":[\"copilot/gpt-5*\",\"openai/gpt-5*\"],\"gpt-5-codex\":[\"copilot/gpt-5*codex*\",\"openai/gpt-5*codex*\"],\"gpt-5-mini\":[\"copilot/gpt-5*mini*\",\"openai/gpt-5*mini*\"],\"gpt-5-nano\":[\"copilot/gpt-5*nano*\",\"openai/gpt-5*nano*\"],\"gpt-5-pro\":[\"copilot/gpt-5*pro*\",\"openai/gpt-5*pro*\"],\"gpt-5.1\":[\"copilot/gpt-5.1*\",\"openai/gpt-5.1*\"],\"gpt-5.2\":[\"copilot/gpt-5.2*\",\"openai/gpt-5.2*\"],\"gpt-5.3\":[\"copilot/gpt-5.3*\",\"openai/gpt-5.3*\"],\"gpt-5.4\":[\"copilot/gpt-5.4*\",\"openai/gpt-5.4*\"],\"gpt-5.5\":[\"copilot/gpt-5.5*\",\"openai/gpt-5.5*\"],\"gpt-5.6\":[\"copilot/gpt-5.6*\",\"openai/gpt-5.6*\"],\"haiku\":[\"copilot/*haiku*\",\"anthropic/*haiku*\"],\"image-generation\":[\"copilot/gpt-image*\",\"openai/gpt-image*\",\"openai/chatgpt-image*\",\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"google/imagen*\"],\"kimi\":[\"copilot/kimi*\",\"openai/kimi*\"],\"kiwi\":[\"copilot/kiwi*\",\"openai/kiwi*\"],\"large\":[\"fable\",\"sonnet\",\"gpt-5-pro\",\"gpt-5\",\"gemini-pro\"],\"lyria\":[\"google/lyria*\",\"gemini/lyria*\",\"copilot/lyria*\"],\"mai-code\":[\"copilot/MAI-Code*\",\"copilot/mai-code*\",\"openai/MAI-Code*\"],\"mai-code-1-flash-picker\":[\"copilot/MAI-Code-1-Flash-picker*\",\"copilot/mai-code-1-flash-picker*\",\"openai/MAI-Code-1-Flash-picker*\"],\"mini\":[\"haiku\",\"gpt-5-mini\",\"gpt-5-nano\",\"gemini-flash-lite\"],\"nano-banana\":[\"copilot/nano-banana*\",\"google/nano-banana*\",\"gemini/nano-banana*\"],\"opus\":[\"copilot/*opus*\",\"anthropic/*opus*\"],\"opusplan\":[\"opus?effort=high\"],\"raptor-mini\":[\"copilot/raptor*\",\"openai/raptor*\"],\"reasoning\":[\"copilot/o1*\",\"copilot/o3*\",\"copilot/o4*\",\"openai/o1*\",\"openai/o3*\",\"openai/o4*\"],\"robotics\":[\"copilot/*robotics*\",\"google/*robotics*\",\"gemini/*robotics*\"],\"small\":[\"mini\"],\"small-agent\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash\"],\"sonnet\":[\"copilot/*sonnet*\",\"anthropic/*sonnet*\"],\"sonnet-6x\":[\"copilot/*sonnet-4.5*\",\"copilot/*sonnet-4.6*\",\"copilot/*sonnet-5*\",\"copilot/*sonnet-4-5-*\",\"anthropic/*sonnet-4-5-*\",\"copilot/*sonnet-4-6*\",\"anthropic/*sonnet-4-6*\",\"anthropic/*sonnet-5*\"],\"summarization\":[\"haiku\",\"gpt-5-mini\",\"gemini-flash-lite\",\"mini\"],\"veo\":[\"google/veo*\",\"gemini/veo*\"],\"vision\":[\"copilot/gemini-*image*\",\"google/gemini-*image*\",\"gemini/gemini-*image*\",\"copilot/gemini-*flash*\",\"google/gemini-*flash*\",\"gemini/gemini-*flash*\"]}},\"container\":{\"imageTag\":\"0.27.42,squid=sha256:42dfeb649c680a8558cd5423dbc530b653a69413e35ffbe5e71da5d48c94bdf0,agent=sha256:26a8af4e5566485b02f52af59ee03803ae798271a9619d4767e94d07806deb9b,agent-act=sha256:a14ad974484aa518aab83d40f3f141175dfd171d3745e01c092375b970f73a20,api-proxy=sha256:944f2686c9ab9bec338fd14b662461662f77cd12cd0ea8a3e7cb8c0987cd1607,cli-proxy=sha256:da006bf96d2d246dd269d57b233c1798d2ad63d6cd64ca02f7bf71045028781f\"},\"logging\":{\"proxyLogsDir\":\"/tmp/gh-aw/sandbox/firewall/logs\",\"auditDir\":\"/tmp/gh-aw/sandbox/firewall/audit\"}}" > "${RUNNER_TEMP}/gh-aw/awf-config.json" + cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json + export GH_AW_MODELS_JSON_PATH="/tmp/gh-aw/models.json" + GH_AW_DOCKER_HOST="" + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_DOCKER_HOST="${DOCKER_HOST}" + fi + if [[ "${DOCKER_HOST:-}" =~ ^tcp:// ]]; then + GH_AW_CHROOT_BINARIES_SOURCE_PATH="${RUNNER_TEMP}/gh-aw" GH_AW_CHROOT_IDENTITY_HOME="${RUNNER_TEMP}/gh-aw/home" node "${RUNNER_TEMP}/gh-aw/actions/patch_awf_chroot_config.cjs" + fi + GH_AW_TOOL_CACHE_MOUNT="" + GH_AW_TOOL_CACHE="${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}" + if [ -d "$GH_AW_TOOL_CACHE" ]; then + if [[ "$GH_AW_TOOL_CACHE" != /opt/* ]]; then + GH_AW_TOOL_CACHE_MOUNT="$GH_AW_TOOL_CACHE:$GH_AW_TOOL_CACHE:ro" + fi + fi + # shellcheck disable=SC1003,SC2016,SC2086 + awf --config "${RUNNER_TEMP}/gh-aw/awf-config.json" --container-workdir "${GITHUB_WORKSPACE}" --mount "${RUNNER_TEMP}/gh-aw:${RUNNER_TEMP}/gh-aw:ro" --mount "${RUNNER_TEMP}/gh-aw:/host${RUNNER_TEMP}/gh-aw:ro" ${GH_AW_TOOL_CACHE_MOUNT:+--mount "$GH_AW_TOOL_CACHE_MOUNT"} ${GH_AW_DOCKER_HOST:+--docker-host "$GH_AW_DOCKER_HOST"} --tty --env-all --exclude-env ANTHROPIC_API_KEY --log-level info --skip-pull \ + -- /bin/bash -c 'set +o histexpand; : "${RUNNER_TOOL_CACHE:?RUNNER_TOOL_CACHE must be set}"; GH_AW_TOOL_CACHE="$RUNNER_TOOL_CACHE"; export PATH="$(find "$GH_AW_TOOL_CACHE" -maxdepth 5 -type d -name bin 2>/dev/null | tr '\''\n'\'' '\'':'\'')$PATH"; [ -n "$GOROOT" ] && export PATH="$GOROOT/bin:$PATH" || true; [ -n "$ERLANG_HOME" ] && export PATH="$ERLANG_HOME/bin:$PATH" || true && GH_AW_NODE_EXEC="${GH_AW_NODE_BIN:-}"; if [ -z "$GH_AW_NODE_EXEC" ] || [ ! -x "$GH_AW_NODE_EXEC" ]; then GH_AW_NODE_EXEC="$(command -v node 2>/dev/null || true)"; fi; if [ -z "$GH_AW_NODE_EXEC" ]; then echo "node runtime missing on this runner — check runtimes.node in workflow YAML" >&2; exit 127; fi; GH_AW_NPM_GLOBAL_ROOT="$(npm root -g 2>/dev/null || true)"; if [ -n "$GH_AW_NPM_GLOBAL_ROOT" ]; then export NODE_PATH="${GH_AW_NPM_GLOBAL_ROOT}${NODE_PATH:+:${NODE_PATH}}"; fi; "$GH_AW_NODE_EXEC" ${RUNNER_TEMP}/gh-aw/actions/claude_harness.cjs claude --print --no-chrome --allowed-tools '\''Bash,BashOutput,Edit(/tmp/*),ExitPlanMode,Glob,Grep,KillBash,LS,MultiEdit(/tmp/*),NotebookRead,Read,Read(/tmp/*),Task,TodoWrite,Write(/tmp/*)'\'' --debug-file /tmp/gh-aw/evals/evals.log --verbose --permission-mode acceptEdits --output-format stream-json --prompt-file /tmp/gh-aw/aw-prompts/prompt.txt' 2>&1 | tee -a /tmp/gh-aw/evals/evals.log + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_MAX_RETRIES: 0 + ANTHROPIC_MODEL: small + BASH_DEFAULT_TIMEOUT_MS: 60000 + BASH_MAX_TIMEOUT_MS: 60000 + CLAUDE_CODE_DISABLE_FAST_MODE: 1 + DISABLE_BUG_COMMAND: 1 + DISABLE_ERROR_REPORTING: 1 + DISABLE_TELEMETRY: 1 + GH_AW_LLM_PROVIDER: anthropic + GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} + GH_AW_PHASE: evals + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_VERSION: dev + GITHUB_AW: true + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + GIT_AUTHOR_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_AUTHOR_NAME: github-actions[bot] + GIT_COMMITTER_EMAIL: github-actions[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: github-actions[bot] + MCP_TIMEOUT: 120000 + MCP_TOOL_TIMEOUT: 60000 + RUNNER_TEMP: ${{ runner.temp }} + TRACEPARENT: ${{ env.GITHUB_AW_OTEL_TRACE_ID != '' && env.GITHUB_AW_OTEL_PARENT_SPAN_ID != '' && format('00-{0}-{1}-01', env.GITHUB_AW_OTEL_TRACE_ID, env.GITHUB_AW_OTEL_PARENT_SPAN_ID) || '' }} + - name: Parse MCP Gateway logs for step summary + if: always() + id: parse-mcp-gateway + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Parse BinEval results + if: always() + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EVALS_QUESTIONS: '[{"id":"tone_style_goal_met","question":"Does the agent output show that the objective for experiment tone_style was successfully completed?"}]' + GH_AW_EVALS_MODEL: "small" + GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/run_evals.cjs'); + await main(); + - name: Redact secrets in evals results + id: redact_evals_results + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/redact_evals_results.cjs'); + await main(); + - name: Render evals results to step summary + if: steps.redact_evals_results.outcome == 'success' + continue-on-error: true + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/render_evals_summary.cjs'); + await main(); + - name: Upload evals results + if: steps.redact_evals_results.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: evals + path: /tmp/gh-aw/evals.jsonl + if-no-files-found: ignore + + push_evals_state: + needs: + - activation + - evals + if: always() && (!cancelled()) && needs.evals.result != 'skipped' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Typist - Go Type Analysis" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/typist.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "2.1.220" + GH_AW_INFO_AWF_VERSION: "v0.27.42" + GH_AW_INFO_ENGINE_ID: "claude" + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download evals artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: evals + path: /tmp/gh-aw/evals-state + - name: Push evals results to git + id: push_evals_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_STATE_DIR: /tmp/gh-aw/evals-state + GH_AW_STATE_BRANCH: evals/typist + GH_AW_STATE_FILES: evals.jsonl + GH_AW_STATE_LABEL: evals results + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + push_experiments_state: needs: activation if: always() && (!cancelled()) && needs.activation.result == 'success' diff --git a/.github/workflows/typist.md b/.github/workflows/typist.md index b629ea23e21..3dfe706436c 100644 --- a/.github/workflows/typist.md +++ b/.github/workflows/typist.md @@ -61,6 +61,10 @@ sandbox: sudo: false features: gh-aw-detection: true +evals: + - id: tone_style_goal_met + question: Does the agent output show that the objective for experiment tone_style was successfully completed? + --- # Typist - Go Type Consistency Analysis diff --git a/docs/src/content/docs/experimental/experiments.md b/docs/src/content/docs/experimental/experiments.md index 974b541310f..dee1403d16b 100644 --- a/docs/src/content/docs/experimental/experiments.md +++ b/docs/src/content/docs/experimental/experiments.md @@ -84,6 +84,9 @@ Summarize the findings in a **${{ experiments.prompt_style }}** way. When `evals` are configured, `metric` can reference an eval question ID using `eval:` (for example `eval:focused`) or `evals.`. +`gh aw experiments analyze ` resolves the referenced eval question and, when +eval result data is available, shows YES/NO/UNKNOWN totals for that eval-backed metric. + > [!NOTE] > Experiment names must be valid identifiers: start with a letter or > underscore, followed by letters, digits, or underscores. For example, use diff --git a/pkg/cli/experiments_analyze_statistics.go b/pkg/cli/experiments_analyze_statistics.go index 6b9ca49d149..5a16cdd88bf 100644 --- a/pkg/cli/experiments_analyze_statistics.go +++ b/pkg/cli/experiments_analyze_statistics.go @@ -42,6 +42,10 @@ type ExperimentAnalysis struct { // Empty when Metric is absent or does not reference an eval. MetricQuestion string `json:"metric_question,omitempty"` + // MetricEvalResults summarizes observed eval outcomes when Metric references an eval ID + // and evals.jsonl data is available. + MetricEvalResults *MetricEvalResults `json:"metric_eval_results,omitempty"` + // MinSamples is the minimum runs per variant required before analysis is reliable. // Defaults to 20 when not declared in the experiment config (R-STAT-007). MinSamples int `json:"min_samples"` @@ -104,10 +108,25 @@ type GuardrailStatus struct { Threshold string `json:"threshold"` } +// MetricEvalResults summarizes observed YES/NO/UNKNOWN outcomes for an eval-backed metric. +type MetricEvalResults struct { + Yes int `json:"yes"` + No int `json:"no"` + Unknown int `json:"unknown"` + Total int `json:"total"` + LatestAnswer string `json:"latest_answer,omitempty"` + LatestRunID string `json:"latest_run_id,omitempty"` +} + // computeExperimentAnalysis computes the statistical analysis for a single named experiment. // cfg may be nil when no workflow frontmatter is available, in which case defaults are used. // evals provides the eval definitions for resolving eval-backed metric references; may be nil. -func computeExperimentAnalysis(exp ExperimentVariantStats, cfg *workflow.ExperimentConfig, evals *workflow.EvalsConfig) ExperimentAnalysis { +func computeExperimentAnalysis( + exp ExperimentVariantStats, + cfg *workflow.ExperimentConfig, + evals *workflow.EvalsConfig, + metricEvalResults map[string]MetricEvalResults, +) ExperimentAnalysis { experimentsStatsLog.Printf("Computing analysis for experiment %q: %d variant(s), %d total runs", exp.Name, len(exp.Variants), exp.Total) a := ExperimentAnalysis{ ExperimentName: exp.Name, @@ -132,11 +151,19 @@ func computeExperimentAnalysis(exp ExperimentVariantStats, cfg *workflow.Experim if cfg.Metric != "" { a.Metric = cfg.Metric evalID, isEval := workflow.ParseExperimentMetricEvalReference(cfg.Metric) - if isEval && evalID != "" && evals != nil { - for _, q := range evals.Questions { - if q.ID == evalID { - a.MetricQuestion = q.Question - break + if isEval && evalID != "" { + if evals != nil { + for _, q := range evals.Questions { + if q.ID == evalID { + a.MetricQuestion = q.Question + break + } + } + } + if metricEvalResults != nil { + if summary, ok := metricEvalResults[evalID]; ok { + s := summary + a.MetricEvalResults = &s } } } @@ -319,6 +346,24 @@ func printOneExperimentAnalysis(a ExperimentAnalysis) { } else { fmt.Fprintf(os.Stderr, " Metric : %s\n", a.Metric) } + if a.MetricEvalResults != nil { + fmt.Fprintf( + os.Stderr, + " Eval : YES=%d NO=%d UNKNOWN=%d TOTAL=%d", + a.MetricEvalResults.Yes, + a.MetricEvalResults.No, + a.MetricEvalResults.Unknown, + a.MetricEvalResults.Total, + ) + if a.MetricEvalResults.LatestAnswer != "" { + fmt.Fprintf(os.Stderr, " (latest: %s", a.MetricEvalResults.LatestAnswer) + if a.MetricEvalResults.LatestRunID != "" { + fmt.Fprintf(os.Stderr, " run %s", a.MetricEvalResults.LatestRunID) + } + fmt.Fprint(os.Stderr, ")") + } + fmt.Fprintln(os.Stderr) + } } fmt.Fprintf(os.Stderr, " Min samples: %d per variant\n", a.MinSamples) diff --git a/pkg/cli/experiments_analyze_statistics_test.go b/pkg/cli/experiments_analyze_statistics_test.go index e3ded98c1d2..4e101dce326 100644 --- a/pkg/cli/experiments_analyze_statistics_test.go +++ b/pkg/cli/experiments_analyze_statistics_test.go @@ -180,7 +180,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: map[string]int{"concise": 5, "detailed": 5}, Total: 10, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Equal(t, "style", a.ExperimentName, "experiment name") assert.Equal(t, defaultMinSamples, a.MinSamples, "default min_samples") @@ -203,7 +203,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: []string{"formal", "casual"}, MinSamples: 5, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.Equal(t, 5, a.MinSamples, "min_samples from config") assert.Equal(t, "READY_FOR_ANALYSIS", a.Recommendation, "count >= min_samples → READY") for _, v := range a.Variants { @@ -223,7 +223,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { AnalysisType: "t_test", MinSamples: 20, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.Equal(t, "H0: no change. H1: short reduces tokens by 15%.", a.Hypothesis, "hypothesis from config") assert.Equal(t, "t_test", a.AnalysisType, "analysis_type from config") assert.Equal(t, "READY_FOR_ANALYSIS", a.Recommendation, "count >= min_samples → READY") @@ -242,7 +242,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { {Name: "empty_output_rate", Threshold: "==0"}, }, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) require.Len(t, a.Guardrails, 2, "should have two guardrails") assert.Equal(t, "success_rate", a.Guardrails[0].Name, "first guardrail name") assert.Equal(t, ">=0.95", a.Guardrails[0].Threshold, "first guardrail threshold") @@ -255,7 +255,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: map[string]int{"A": 5, "B": 5, "C": 5}, Total: 15, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Len(t, a.Variants, 3, "three variants") // α_adjusted = 0.05 / (K − 1) = 0.05 / 2 = 0.025 assert.InDelta(t, 0.025, a.BonferroniAlpha, 0.0001, "Bonferroni alpha for K=3") @@ -268,7 +268,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: map[string]int{"A": 5, "B": 5, "C": 5, "D": 5}, Total: 20, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) // α_adjusted = 0.05 / (4 − 1) = 0.05 / 3 ≈ 0.0167 assert.InDelta(t, 0.05/3.0, a.BonferroniAlpha, 0.0001, "Bonferroni alpha for K=4") }) @@ -280,7 +280,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: map[string]int{"A": 19, "B": 1}, Total: 20, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.False(t, a.IsBalanced, "extreme imbalance should not be balanced") assert.Less(t, a.PValue, balanceSignificanceThreshold, "p < 0.05 for extreme imbalance") }) @@ -291,7 +291,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: map[string]int{"A": 0, "B": 0}, Total: 0, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Equal(t, "EXTEND", a.Recommendation, "zero runs → EXTEND") assert.True(t, a.IsBalanced, "insufficient data → default to balanced") assert.InDelta(t, 0.0, a.ChiSquare, 1e-10, "no chi-square for zero total") @@ -303,7 +303,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: map[string]int{"z_last": 3, "a_first": 7}, Total: 10, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) require.Len(t, a.Variants, 2, "two variants") assert.Equal(t, "a_first", a.Variants[0].Name, "first variant alphabetically") assert.Equal(t, "z_last", a.Variants[1].Name, "second variant alphabetically") @@ -320,7 +320,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: []string{"control", "variant"}, Weight: []int{70, 30}, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.True(t, a.IsBalanced, "70/30 split with 70/30 weights should be balanced") // Expected proportions: control=0.7, variant=0.3 require.Len(t, a.Variants, 2, "two variants") @@ -338,7 +338,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: []string{"A", "B"}, Metric: "effective_tokens", } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.Equal(t, "effective_tokens", a.Metric, "metric should be set") assert.Empty(t, a.MetricQuestion, "MetricQuestion empty for plain metric") }) @@ -359,7 +359,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { {ID: "tests", Question: "Do the tests pass?"}, }, } - a := computeExperimentAnalysis(exp, cfg, evals) + a := computeExperimentAnalysis(exp, cfg, evals, nil) assert.Equal(t, "evals.builds", a.Metric, "metric set to original reference") assert.Equal(t, "Does the generated code compile?", a.MetricQuestion, "eval question resolved") }) @@ -379,11 +379,46 @@ func TestComputeExperimentAnalysis(t *testing.T) { {ID: "tests", Question: "Do the tests pass?"}, }, } - a := computeExperimentAnalysis(exp, cfg, evals) + a := computeExperimentAnalysis(exp, cfg, evals, nil) assert.Equal(t, "eval:tests", a.Metric, "metric set to original reference") assert.Equal(t, "Do the tests pass?", a.MetricQuestion, "eval question resolved via eval: prefix") }) + t.Run("metric eval reference includes eval result summary when available", func(t *testing.T) { + exp := ExperimentVariantStats{ + Name: "quality", + Variants: map[string]int{"A": 8, "B": 12}, + Total: 20, + } + cfg := &workflow.ExperimentConfig{ + Variants: []string{"A", "B"}, + Metric: "evals.builds", + } + evals := &workflow.EvalsConfig{ + Questions: []workflow.EvalDefinition{ + {ID: "builds", Question: "Does the generated code compile?"}, + }, + } + metricResults := map[string]MetricEvalResults{ + "builds": { + Yes: 12, + No: 5, + Unknown: 3, + Total: 20, + LatestAnswer: "YES", + LatestRunID: "123456", + }, + } + a := computeExperimentAnalysis(exp, cfg, evals, metricResults) + require.NotNil(t, a.MetricEvalResults, "eval result summary should be attached") + assert.Equal(t, 12, a.MetricEvalResults.Yes) + assert.Equal(t, 5, a.MetricEvalResults.No) + assert.Equal(t, 3, a.MetricEvalResults.Unknown) + assert.Equal(t, 20, a.MetricEvalResults.Total) + assert.Equal(t, "YES", a.MetricEvalResults.LatestAnswer) + assert.Equal(t, "123456", a.MetricEvalResults.LatestRunID) + }) + t.Run("eval reference with unknown id leaves MetricQuestion empty", func(t *testing.T) { exp := ExperimentVariantStats{ Name: "x", @@ -399,7 +434,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { {ID: "builds", Question: "Does the generated code compile?"}, }, } - a := computeExperimentAnalysis(exp, cfg, evals) + a := computeExperimentAnalysis(exp, cfg, evals, nil) assert.Equal(t, "evals.unknown_id", a.Metric, "metric preserved") assert.Empty(t, a.MetricQuestion, "MetricQuestion empty when eval id not found") }) @@ -414,7 +449,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { Variants: []string{"A", "B"}, Metric: "evals.builds", } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.Equal(t, "evals.builds", a.Metric, "metric preserved") assert.Empty(t, a.MetricQuestion, "MetricQuestion empty when evals is nil") }) @@ -434,7 +469,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { {ID: "builds", Question: "Does the generated code compile?"}, }, } - a := computeExperimentAnalysis(exp, cfg, evals) + a := computeExperimentAnalysis(exp, cfg, evals, nil) assert.Equal(t, "evals.builds", a.Metric, "metric preserved for single-variant state") assert.Equal(t, "Does the generated code compile?", a.MetricQuestion, "metric question resolved before degenerate return") assert.Equal(t, "EXTEND", a.Recommendation, "single-variant state still extends") @@ -444,7 +479,7 @@ func TestComputeExperimentAnalysis(t *testing.T) { // TestComputeExperimentAnalyses tests the bulk analysis function. func TestComputeExperimentAnalyses(t *testing.T) { t.Run("empty experiments returns nil", func(t *testing.T) { - result := computeExperimentAnalyses(nil, nil, nil) + result := computeExperimentAnalyses(nil, nil, nil, nil) assert.Nil(t, result, "nil experiments should return nil") }) @@ -453,7 +488,7 @@ func TestComputeExperimentAnalyses(t *testing.T) { {Name: "exp1", Variants: map[string]int{"A": 5, "B": 5}, Total: 10}, {Name: "exp2", Variants: map[string]int{"X": 3, "Y": 7}, Total: 10}, } - analyses := computeExperimentAnalyses(experiments, nil, nil) + analyses := computeExperimentAnalyses(experiments, nil, nil, nil) require.Len(t, analyses, 2, "should produce one analysis per experiment") assert.Equal(t, "exp1", analyses[0].ExperimentName, "first analysis name") assert.Equal(t, "exp2", analyses[1].ExperimentName, "second analysis name") @@ -471,7 +506,7 @@ func TestComputeExperimentAnalyses(t *testing.T) { MinSamples: 20, }, } - analyses := computeExperimentAnalyses(experiments, configs, nil) + analyses := computeExperimentAnalyses(experiments, configs, nil, nil) require.Len(t, analyses, 1, "one analysis") assert.Equal(t, "test hypothesis", analyses[0].Hypothesis, "hypothesis from config") assert.Equal(t, "proportion_test", analyses[0].AnalysisType, "analysis type from config") @@ -496,7 +531,7 @@ func TestExperimentAnalysisJSONOutput(t *testing.T) { }, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) jsonBytes, err := json.MarshalIndent(a, "", " ") require.NoError(t, err, "should marshal analysis to JSON") @@ -525,7 +560,7 @@ func TestExperimentAnalysisBonferroniAbsent(t *testing.T) { Variants: map[string]int{"yes": 10, "no": 10}, Total: 20, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.InDelta(t, 0.0, a.BonferroniAlpha, 1e-10, "BonferroniAlpha should be zero for K=2") jsonBytes, err := json.MarshalIndent(a, "", " ") @@ -544,7 +579,7 @@ func TestMinSamplesDefaultApplied(t *testing.T) { Variants: map[string]int{"A": 10, "B": 10}, Total: 20, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Equal(t, defaultMinSamples, a.MinSamples, "default min_samples should be 20") } @@ -581,7 +616,7 @@ func TestObservedPctSumsToHundred(t *testing.T) { Variants: map[string]int{"A": 7, "B": 13}, Total: 20, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) total := 0.0 for _, v := range a.Variants { total += v.ObservedPct @@ -611,7 +646,7 @@ func TestExpectedPctSumsToHundred(t *testing.T) { total += c } exp := ExperimentVariantStats{Name: "e", Variants: tt.variants, Total: total} - a := computeExperimentAnalysis(exp, tt.cfg, nil) + a := computeExperimentAnalysis(exp, tt.cfg, nil, nil) sum := 0.0 for _, v := range a.Variants { sum += v.ExpectedPct @@ -632,7 +667,7 @@ func TestReadyForAnalysisAllAboveMinSamples(t *testing.T) { Variants: []string{"X", "Y"}, MinSamples: 20, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.Equal(t, "READY_FOR_ANALYSIS", a.Recommendation, "all variants above min_samples → READY") assert.Contains(t, a.Rationale, "min_samples", "rationale should mention min_samples") @@ -652,7 +687,7 @@ func TestPartiallyBelowMinSamples(t *testing.T) { Variants: []string{"above", "below"}, MinSamples: 20, } - a := computeExperimentAnalysis(exp, cfg, nil) + a := computeExperimentAnalysis(exp, cfg, nil, nil) assert.Equal(t, "EXTEND", a.Recommendation, "one variant below threshold → EXTEND") assert.Contains(t, a.Rationale, "1 of 2", "rationale should count variants below threshold") @@ -671,7 +706,7 @@ func TestChiSquarePerfectBalance(t *testing.T) { Variants: map[string]int{"A": 10, "B": 10, "C": 10}, Total: 30, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.InDelta(t, 0.0, a.ChiSquare, 1e-10, "chi² should be 0 for perfectly balanced sample") assert.InDelta(t, 1.0, a.PValue, 1e-10, "p should be 1.0 for chi²=0") assert.True(t, a.IsBalanced, "perfectly balanced → is_balanced") @@ -754,7 +789,7 @@ func TestAnalysisWithNilConfig(t *testing.T) { Variants: map[string]int{"on": 8, "off": 12}, Total: 20, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Equal(t, "no_config", a.ExperimentName, "experiment name preserved") assert.Empty(t, a.Hypothesis, "no hypothesis without config") assert.Empty(t, a.AnalysisType, "no analysis type without config") @@ -775,7 +810,7 @@ func TestComputeExperimentAnalysisDegenerateVariants(t *testing.T) { Variants: map[string]int{}, Total: 0, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Equal(t, "EXTEND", a.Recommendation, "zero variants → EXTEND") assert.True(t, a.IsBalanced, "degenerate case defaults to balanced") assert.Empty(t, a.Variants, "no variant entries") @@ -788,7 +823,7 @@ func TestComputeExperimentAnalysisDegenerateVariants(t *testing.T) { Variants: map[string]int{"only": 10}, Total: 10, } - a := computeExperimentAnalysis(exp, nil, nil) + a := computeExperimentAnalysis(exp, nil, nil, nil) assert.Equal(t, "EXTEND", a.Recommendation, "single variant → EXTEND") assert.True(t, a.IsBalanced, "degenerate case defaults to balanced") assert.Empty(t, a.Variants, "no variant entries emitted for degenerate case") diff --git a/pkg/cli/experiments_command.go b/pkg/cli/experiments_command.go index 3b22f34c578..4f7fec221a3 100644 --- a/pkg/cli/experiments_command.go +++ b/pkg/cli/experiments_command.go @@ -1,6 +1,7 @@ package cli import ( + "bufio" "encoding/base64" "encoding/json" "errors" @@ -264,8 +265,20 @@ func RunExperimentsAnalyze(config ExperimentsAnalyzeConfig) error { return nil } + var metricEvalResults map[string]MetricEvalResults + if config.RepoOverride != "" { + metricEvalResults = loadRemoteMetricEvalResults(config.RepoOverride, details.WorkflowID) + } else { + metricEvalResults = loadLocalMetricEvalResults(details.WorkflowID) + } + // Compute statistical analyses for each named experiment. - details.Analyses = computeExperimentAnalyses(details.Experiments, frontmatterResult.ExperimentConfigs, frontmatterResult.Evals) + details.Analyses = computeExperimentAnalyses( + details.Experiments, + frontmatterResult.ExperimentConfigs, + frontmatterResult.Evals, + metricEvalResults, + ) if config.JSONOutput { jsonBytes, err := json.MarshalIndent(details, "", " ") @@ -283,7 +296,12 @@ func RunExperimentsAnalyze(config ExperimentsAnalyzeConfig) error { // computeExperimentAnalyses computes statistical analyses for all named experiments. // configs maps experiment names to their configuration; values may be nil. // evals provides the eval definitions for resolving eval-backed metric references; may be nil. -func computeExperimentAnalyses(experiments []ExperimentVariantStats, configs map[string]*workflow.ExperimentConfig, evals *workflow.EvalsConfig) []ExperimentAnalysis { +func computeExperimentAnalyses( + experiments []ExperimentVariantStats, + configs map[string]*workflow.ExperimentConfig, + evals *workflow.EvalsConfig, + metricEvalResults map[string]MetricEvalResults, +) []ExperimentAnalysis { if len(experiments) == 0 { return nil } @@ -293,11 +311,79 @@ func computeExperimentAnalyses(experiments []ExperimentVariantStats, configs map if configs != nil { cfg = configs[exp.Name] } - analyses = append(analyses, computeExperimentAnalysis(exp, cfg, evals)) + analyses = append(analyses, computeExperimentAnalysis(exp, cfg, evals, metricEvalResults)) } return analyses } +type evalResultRecord struct { + ID string `json:"id"` + Answer string `json:"answer"` + RunID string `json:"runid"` + Timestamp string `json:"timestamp"` +} + +func loadLocalMetricEvalResults(workflowID string) map[string]MetricEvalResults { + branchName := workflow.WorkflowStateBranchName(constants.EvalsBranchPrefix, workflowID) + ref := "origin/" + branchName + if !gitRefExists(ref) { + if !gitRefExists(branchName) { + return nil + } + ref = branchName + } + cmd := exec.Command("git", "show", ref+":"+constants.EvalsResultFilename) + out, err := cmd.Output() + if err != nil { + return nil + } + return summarizeMetricEvalResults(out) +} + +func loadRemoteMetricEvalResults(repoOverride, workflowID string) map[string]MetricEvalResults { + branchName := workflow.WorkflowStateBranchName(constants.EvalsBranchPrefix, workflowID) + decoded, err := readRemoteRepoBranchFile(repoOverride, branchName, constants.EvalsResultFilename, "") + if err != nil { + return nil + } + return summarizeMetricEvalResults(decoded) +} + +func summarizeMetricEvalResults(data []byte) map[string]MetricEvalResults { + scanner := bufio.NewScanner(strings.NewReader(string(data))) + results := map[string]MetricEvalResults{} + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var record evalResultRecord + if err := json.Unmarshal([]byte(line), &record); err != nil { + continue + } + if record.ID == "" { + continue + } + summary := results[record.ID] + summary.Total++ + switch strings.ToUpper(strings.TrimSpace(record.Answer)) { + case "YES": + summary.Yes++ + case "NO": + summary.No++ + default: + summary.Unknown++ + } + summary.LatestAnswer = strings.ToUpper(strings.TrimSpace(record.Answer)) + summary.LatestRunID = record.RunID + results[record.ID] = summary + } + if len(results) == 0 { + return nil + } + return results +} + // experimentFrontmatterResult holds both the experiment configs and evals config parsed // from a workflow's frontmatter. type experimentFrontmatterResult struct { diff --git a/pkg/cli/experiments_command_test.go b/pkg/cli/experiments_command_test.go index 2498bdd72b5..5c35f7d315c 100644 --- a/pkg/cli/experiments_command_test.go +++ b/pkg/cli/experiments_command_test.go @@ -236,6 +236,35 @@ func TestParsePagedJSONArray(t *testing.T) { } } +func TestSummarizeMetricEvalResults(t *testing.T) { + data := []byte(`{"id":"quality","answer":"YES","runid":"100"} +{"id":"quality","answer":"NO","runid":"101"} +{"id":"quality","answer":"UNKNOWN","runid":"102"} +{"id":"quality","answer":"MAYBE","runid":"103"} +{"id":"coverage","answer":"YES","runid":"200"} +`) + result := summarizeMetricEvalResults(data) + require.NotNil(t, result) + + quality, ok := result["quality"] + require.True(t, ok) + assert.Equal(t, 1, quality.Yes) + assert.Equal(t, 1, quality.No) + assert.Equal(t, 2, quality.Unknown) + assert.Equal(t, 4, quality.Total) + assert.Equal(t, "MAYBE", quality.LatestAnswer) + assert.Equal(t, "103", quality.LatestRunID) + + coverage, ok := result["coverage"] + require.True(t, ok) + assert.Equal(t, 1, coverage.Yes) + assert.Equal(t, 0, coverage.No) + assert.Equal(t, 0, coverage.Unknown) + assert.Equal(t, 1, coverage.Total) + assert.Equal(t, "YES", coverage.LatestAnswer) + assert.Equal(t, "200", coverage.LatestRunID) +} + func TestExperimentInfoJSONOutput(t *testing.T) { experiments := []ExperimentInfo{ { From 917bfe266c65bc0700b422ef16dd019599cf6b10 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 29 Jul 2026 06:42:13 +0000 Subject: [PATCH 2/2] Guard linter shared indexes against nil analysis package in tests Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/linters/internal/filecheck/filecheck.go | 6 +++++- pkg/linters/internal/nolint/nolint.go | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/linters/internal/filecheck/filecheck.go b/pkg/linters/internal/filecheck/filecheck.go index 297bafbcd19..322529cb3b6 100644 --- a/pkg/linters/internal/filecheck/filecheck.go +++ b/pkg/linters/internal/filecheck/filecheck.go @@ -40,7 +40,11 @@ func Index(pass *analysis.Pass) (GeneratedIndex, error) { // BuildGeneratedIndex returns the set of generated Go source files in pass. func BuildGeneratedIndex(pass *analysis.Pass) GeneratedIndex { - pkgLog.Printf("building generated-file index for %s (%d files)", pass.Pkg.Path(), len(pass.Files)) + pkgPath := "" + if pass.Pkg != nil { + pkgPath = pass.Pkg.Path() + } + pkgLog.Printf("building generated-file index for %s (%d files)", pkgPath, len(pass.Files)) generated := make(GeneratedIndex) for _, file := range pass.Files { if !ast.IsGenerated(file) { diff --git a/pkg/linters/internal/nolint/nolint.go b/pkg/linters/internal/nolint/nolint.go index 2b42588a0bd..1e03ce1ea46 100644 --- a/pkg/linters/internal/nolint/nolint.go +++ b/pkg/linters/internal/nolint/nolint.go @@ -43,7 +43,11 @@ func Index(pass *analysis.Pass) (DirectiveIndex, error) { // BuildDirectiveIndex scans all comments in the analysis pass and returns a map // from filename → line → set of linter names that carry a nolint directive. func BuildDirectiveIndex(pass *analysis.Pass) DirectiveIndex { - pkgLog.Printf("building nolint directive index for %s (%d files)", pass.Pkg.Path(), len(pass.Files)) + pkgPath := "" + if pass.Pkg != nil { + pkgPath = pass.Pkg.Path() + } + pkgLog.Printf("building nolint directive index for %s (%d files)", pkgPath, len(pass.Files)) noLintLinesByFile := make(DirectiveIndex, len(pass.Files)) for _, file := range pass.Files { filename := pass.Fset.PositionFor(file.Pos(), false).Filename