From e271ccb77ec7e4ef871201f74453326e30ae656d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 5 Jul 2026 13:21:26 +0000 Subject: [PATCH 1/2] Initial plan From 6542a21afd162c6632007c2fc68e01a43fc100bd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 5 Jul 2026 13:31:39 +0000 Subject: [PATCH 2/2] Add model_size A/B experiment to test-quality-sentinel workflow Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../workflows/test-quality-sentinel.lock.yml | 130 +++++++++++++++++- .github/workflows/test-quality-sentinel.md | 16 +++ 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-quality-sentinel.lock.yml b/.github/workflows/test-quality-sentinel.lock.yml index a626a2953ab..430f261f350 100644 --- a/.github/workflows/test-quality-sentinel.lock.yml +++ b/.github/workflows/test-quality-sentinel.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"c093d2c0c371250dcf788a5fbea8e5e7b2126deffae1de2c5f1c7314f434ae73","body_hash":"d8b4870a9a2fd8e4adda12053af38a9d1cb46f7f6d4d45bfaedb88aef8d18a88","strict":true,"agent_id":"copilot","engine_versions":{"copilot":"1.0.68"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"53ba1bb0cf65b10cd8b369840e36054a5f6053553b3c1bfdd8d2dc5c5a281941","body_hash":"d8b4870a9a2fd8e4adda12053af38a9d1cb46f7f6d4d45bfaedb88aef8d18a88","strict":true,"agent_id":"copilot","agent_model":"${{ needs.activation.outputs.model_size }}","engine_versions":{"copilot":"1.0.68"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GH_AW_OTEL_GRAFANA_AUTHORIZATION","GH_AW_OTEL_GRAFANA_ENDPOINT","GH_AW_OTEL_SENTRY_AUTHORIZATION","GH_AW_OTEL_SENTRY_ENDPOINT","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.22","digest":"sha256:55f06588411008b7148eb64b8dfe28602a0cce3675b36c6b190b54aca138468e","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.22@sha256:55f06588411008b7148eb64b8dfe28602a0cce3675b36c6b190b54aca138468e"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22","digest":"sha256:afb9ff9140b17d38871dfb9dbac5ff8689ea634c2f91c435da2825192d4881c1","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.22@sha256:afb9ff9140b17d38871dfb9dbac5ff8689ea634c2f91c435da2825192d4881c1"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.22","digest":"sha256:3cdcc1e2b4b4fe602ba69fd3e21aac7ac512d5c1fce24df4ce69dc4f98164b59","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.22@sha256:3cdcc1e2b4b4fe602ba69fd3e21aac7ac512d5c1fce24df4ce69dc4f98164b59"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.33","digest":"sha256:f0e1f1d562f01e737f3ef125cb4d7057ff87a68a97927f5ea9fd6d3f5091da06","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.33@sha256:f0e1f1d562f01e737f3ef125cb4d7057ff87a68a97927f5ea9fd6d3f5091da06"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -107,8 +107,10 @@ jobs: daily_ai_credits_threshold: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_threshold || '' }} daily_ai_credits_total_effective_tokens: ${{ steps.daily-effective-workflow-guardrail.outputs.daily_ai_credits_total_effective_tokens || '' }} engine_id: ${{ steps.generate_aw_info.outputs.engine_id }} + experiments: ${{ steps.pick-experiment.outputs.experiments }} lockdown_check_failed: ${{ steps.generate_aw_info.outputs.lockdown_check_failed == 'true' }} model: ${{ steps.generate_aw_info.outputs.model }} + model_size: ${{ steps.pick-experiment.outputs.model_size }} setup-parent-span-id: ${{ steps.setup.outputs.parent-span-id || steps.setup.outputs.span-id }} setup-span-id: ${{ steps.setup.outputs.span-id }} setup-trace-id: ${{ steps.setup.outputs.trace-id }} @@ -147,7 +149,7 @@ jobs: env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + GH_AW_INFO_MODEL: "${{ needs.activation.outputs.model_size }}" GH_AW_INFO_VERSION: "1.0.68" GH_AW_INFO_AGENT_VERSION: "1.0.68" GH_AW_INFO_WORKFLOW_NAME: "Test Quality Sentinel" @@ -288,10 +290,45 @@ jobs: - name: Log runtime features if: ${{ contains(toJSON(vars), '"GH_AW_RUNTIME_FEATURES":') }} run: bash "${RUNNER_TEMP}/gh-aw/actions/log_runtime_features_summary.sh" + - name: Restore experiment state from git + id: restore-experiment-state + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + GH_AW_EXPERIMENT_BRANCH: experiments/testqualitysentinel + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/load_experiment_state_from_repo.cjs'); + await main(); + - name: Pick experiment variants + id: pick-experiment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_AW_EXPERIMENT_SPEC: '{"model_size":{"variants":["small","large"],"description":"Tests whether a smaller model can preserve test-review decision quality at lower cost versus a larger reasoning-capable model.","hypothesis":"H0: model-size variant does not improve review usefulness acceptance rate. H1: a larger reasoning-capable model improves review usefulness acceptance rate by \u003e=15 percentage points without materially increasing false-positive change requests.","metric":"review_usefulness_acceptance_rate","secondary_metrics":["ai_credits_total","false_positive_change_request_rate"],"guardrail_metrics":[{"name":"false_positive_change_request_rate","threshold":"\u003c=0.20"},{"name":"run_success_rate","threshold":"\u003e=0.90"}],"min_samples":70,"weight":[50,50],"start_date":"2026-07-05"}}' + GH_AW_EXPERIMENT_STATE_FILE: /tmp/gh-aw/experiments/state.json + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/pick_experiment.cjs'); + await main(); + - name: Upload experiment artifact + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: testqualitysentinel-experiment + path: /tmp/gh-aw/experiments + if-no-files-found: ignore + retention-days: 30 - name: Create prompt with built-in context env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt GH_AW_SAFE_OUTPUTS: ${{ runner.temp }}/gh-aw/safeoutputs/outputs.jsonl + GH_AW_EXPERIMENTS_MODEL_SIZE: ${{ steps.pick-experiment.outputs.model_size }} GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -370,6 +407,7 @@ jobs: GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} GH_AW_GITHUB_EVENT_PULL_REQUEST_TITLE: ${{ github.event.pull_request.title }} GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_EXPERIMENTS_MODEL_SIZE: ${{ steps.pick-experiment.outputs.model_size }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); @@ -380,6 +418,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_EXPERIMENTS_MODEL_SIZE: ${{ steps.pick-experiment.outputs.model_size }} GH_AW_EXPR_1A3A194A: ${{ github.event.discussion.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'discussion' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_463A214A: ${{ github.event.pull_request.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'pull_request' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} GH_AW_EXPR_802A9F6A: ${{ github.event.issue.number || (fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_type == 'issue' && fromJSON(github.event.inputs.aw_context || github.event.client_payload.aw_context || '{}').item_number) }} @@ -405,6 +444,7 @@ jobs: return await substitutePlaceholders({ file: process.env.GH_AW_PROMPT, substitutions: { + GH_AW_EXPERIMENTS_MODEL_SIZE: process.env.GH_AW_EXPERIMENTS_MODEL_SIZE, GH_AW_EXPR_1A3A194A: process.env.GH_AW_EXPR_1A3A194A, GH_AW_EXPR_463A214A: process.env.GH_AW_EXPR_463A214A, GH_AW_EXPR_802A9F6A: process.env.GH_AW_EXPR_802A9F6A, @@ -936,7 +976,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }} + COPILOT_MODEL: ${{ needs.activation.outputs.model_size }} GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_AI_CREDITS || '1000' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} @@ -1126,6 +1166,7 @@ jobs: - activation - agent - detection + - push_experiments_state - safe_outputs if: > always() && (needs.agent.result != 'skipped' || needs.activation.outputs.lockdown_check_failed == 'true' || @@ -1461,6 +1502,12 @@ jobs: mkdir -p /tmp/gh-aw/ find "/tmp/gh-aw/" -type f -print echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/agent_output.json" >> "$GITHUB_OUTPUT" + - name: Download experiment artifact + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: testqualitysentinel-experiment + path: /tmp/gh-aw/experiments/ - name: Checkout repository for patch context if: needs.agent.outputs.has_patch == 'true' uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -1691,6 +1738,81 @@ jobs: const { main } = require('${{ runner.temp }}/gh-aw/actions/check_command_position.cjs'); await main(); + push_experiments_state: + needs: activation + if: always() && (!cancelled()) && needs.activation.result == 'success' + runs-on: ubuntu-slim + permissions: + contents: write + env: + GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} + steps: + - name: Checkout actions folder + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: github/gh-aw + sparse-checkout: | + actions + clean: false + persist-credentials: false + - name: Setup Scripts + id: setup + uses: ./actions/setup + with: + destination: ${{ runner.temp }}/gh-aw/actions + job-name: ${{ github.job }} + trace-id: ${{ needs.activation.outputs.setup-trace-id }} + parent-span-id: ${{ needs.activation.outputs.setup-parent-span-id || needs.activation.outputs.setup-span-id }} + env: + GH_AW_SETUP_WORKFLOW_NAME: "Test Quality Sentinel" + GH_AW_CURRENT_WORKFLOW_REF: ${{ github.repository }}/.github/workflows/test-quality-sentinel.lock.yml@${{ github.ref }} + GH_AW_INFO_VERSION: "1.0.68" + GH_AW_INFO_AWF_VERSION: "v0.27.22" + GH_AW_INFO_ENGINE_ID: "copilot" + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: . + - name: Configure Git credentials + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_TOKEN: ${{ github.token }} + run: bash "${RUNNER_TEMP}/gh-aw/actions/configure_git_credentials.sh" + - name: Download experiment artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: testqualitysentinel-experiment + path: /tmp/gh-aw/experiments + - name: Push experiment state to git + id: push_experiments_state + if: always() + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GH_TOKEN: ${{ github.token }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GH_AW_EXPERIMENT_STATE_DIR: /tmp/gh-aw/experiments + GH_AW_EXPERIMENT_BRANCH: experiments/testqualitysentinel + with: + script: | + const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io, getOctokit); + const { main } = require('${{ runner.temp }}/gh-aw/actions/push_experiment_state.cjs'); + await main(); + - name: Restore actions folder + if: always() + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: github/gh-aw + sparse-checkout: | + actions/setup + sparse-checkout-cone-mode: true + clean: false + persist-credentials: false + safe_outputs: needs: - activation @@ -1713,7 +1835,7 @@ jobs: GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: ${{ needs.agent.outputs.model }} + GH_AW_ENGINE_MODEL: "${{ needs.activation.outputs.model_size }}" GH_AW_ENGINE_VERSION: "1.0.68" GH_AW_PROJECT_UTC: "-08:00" GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} diff --git a/.github/workflows/test-quality-sentinel.md b/.github/workflows/test-quality-sentinel.md index 06fd113b799..2614244e669 100644 --- a/.github/workflows/test-quality-sentinel.md +++ b/.github/workflows/test-quality-sentinel.md @@ -16,6 +16,7 @@ permissions: copilot-requests: write engine: id: copilot + model: "${{ needs.activation.outputs.model_size }}" max-continuations: 15 tools: cli-proxy: true @@ -125,6 +126,21 @@ imports: - shared/otlp.md features: gh-aw-detection: true +experiments: + model_size: + variants: [small, large] + description: "Tests whether a smaller model can preserve test-review decision quality at lower cost versus a larger reasoning-capable model." + hypothesis: "H0: model-size variant does not improve review usefulness acceptance rate. H1: a larger reasoning-capable model improves review usefulness acceptance rate by >=15 percentage points without materially increasing false-positive change requests." + metric: review_usefulness_acceptance_rate + secondary_metrics: [ai_credits_total, false_positive_change_request_rate] + guardrail_metrics: + - name: false_positive_change_request_rate + threshold: "<=0.20" + - name: run_success_rate + threshold: ">=0.90" + min_samples: 70 + weight: [50, 50] + start_date: "2026-07-05" --- # Test Quality Sentinel 🧪