diff --git a/.github/workflows/smoke-copilot-byok-aoai-apikey.lock.yml b/.github/workflows/smoke-copilot-byok-aoai-apikey.lock.yml index de59a82f..2f7ef60c 100644 --- a/.github/workflows/smoke-copilot-byok-aoai-apikey.lock.yml +++ b/.github/workflows/smoke-copilot-byok-aoai-apikey.lock.yml @@ -898,7 +898,7 @@ jobs: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }} + COPILOT_MODEL: ${{ env.COPILOT_MODEL }} COPILOT_PROVIDER_API_KEY: ${{ secrets.FOUNDRY_API_KEY }} COPILOT_PROVIDER_BASE_URL: ${{ secrets.FOUNDRY_OPENAI_ENDPOINT }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} diff --git a/.github/workflows/smoke-copilot-byok-aoai-entra.lock.yml b/.github/workflows/smoke-copilot-byok-aoai-entra.lock.yml index 89c144c8..6439f1b2 100644 --- a/.github/workflows/smoke-copilot-byok-aoai-entra.lock.yml +++ b/.github/workflows/smoke-copilot-byok-aoai-entra.lock.yml @@ -880,7 +880,7 @@ jobs: AWF_REFLECT_ENABLED: 1 COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }} + COPILOT_MODEL: ${{ env.COPILOT_MODEL }} COPILOT_PROVIDER_BASE_URL: ${{ secrets.FOUNDRY_OPENAI_ENDPOINT }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json diff --git a/.github/workflows/smoke-copilot-byok.lock.yml b/.github/workflows/smoke-copilot-byok.lock.yml index 98171557..328c469f 100644 --- a/.github/workflows/smoke-copilot-byok.lock.yml +++ b/.github/workflows/smoke-copilot-byok.lock.yml @@ -869,7 +869,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }} + COPILOT_MODEL: ${{ env.COPILOT_MODEL }} COPILOT_PROVIDER_API_KEY: ${{ secrets.COPILOT_GITHUB_TOKEN }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json diff --git a/scripts/ci/postprocess-smoke-workflows.test.ts b/scripts/ci/postprocess-smoke-workflows.test.ts index 5e9867ae..d802a472 100644 --- a/scripts/ci/postprocess-smoke-workflows.test.ts +++ b/scripts/ci/postprocess-smoke-workflows.test.ts @@ -291,8 +291,8 @@ const legacyApiProxyLogsDirRegex = const copySessionStateStepRegex = /^(\s+)- name: Copy Copilot session state files to logs\n\1 if: always\(\)\n\1 continue-on-error: true\n\1 run: bash "\$\{RUNNER_TEMP\}\/gh-aw\/actions\/copy_copilot_session_state\.sh"\n/m; -const copilotModelEmptyFallbackRegex = - /(COPILOT_MODEL:\s*\$\{\{\s*vars\.GH_AW_MODEL_AGENT_COPILOT\s*\|\|\s*)(?:vars\.GH_AW_DEFAULT_MODEL_COPILOT\s*\|\|\s*)?'[^']*'(\s*\}\})/g; +const copilotModelOverrideRegex = + /^(\s*COPILOT_MODEL:\s*)\$\{\{\s*(?:vars\.GH_AW_MODEL_AGENT_COPILOT\s*\|\|\s*)?(?:vars\.GH_AW_DEFAULT_MODEL_COPILOT\s*\|\|\s*)?(?:env\.COPILOT_MODEL|''|'[^']*')\s*\}\}[ \t]*$/gm; function buildCopySessionStateStep(indent: string): string { const i = indent; @@ -429,31 +429,31 @@ describe('buildCopySessionStateStep', () => { }); }); -describe('copilotModelEmptyFallbackRegex', () => { +describe('copilotModelOverrideRegex', () => { beforeEach(() => { - copilotModelEmptyFallbackRegex.lastIndex = 0; + copilotModelOverrideRegex.lastIndex = 0; }); - it('should replace empty fallback with env.COPILOT_MODEL', () => { + it('should replace empty fallback with workflow-level env.COPILOT_MODEL', () => { const input = " COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }}\n"; const result = input.replace( - copilotModelEmptyFallbackRegex, - `$1env.COPILOT_MODEL$2` + copilotModelOverrideRegex, + '$1${{ env.COPILOT_MODEL }}' ); expect(result).toBe( - ` COPILOT_MODEL: \${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }}\n` + ` COPILOT_MODEL: \${{ env.COPILOT_MODEL }}\n` ); }); - it('should replace hardcoded model fallback with env.COPILOT_MODEL', () => { + it('should replace hardcoded model fallback with workflow-level env.COPILOT_MODEL', () => { const input = " COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || 'claude-opus-4.8' }}\n"; const result = input.replace( - copilotModelEmptyFallbackRegex, - `$1env.COPILOT_MODEL$2` + copilotModelOverrideRegex, + '$1${{ env.COPILOT_MODEL }}' ); expect(result).toBe( - ` COPILOT_MODEL: \${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }}\n` + ` COPILOT_MODEL: \${{ env.COPILOT_MODEL }}\n` ); }); @@ -461,20 +461,29 @@ describe('copilotModelEmptyFallbackRegex', () => { const input = " COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || vars.GH_AW_DEFAULT_MODEL_COPILOT || 'claude-sonnet-4.6' }}\n"; const result = input.replace( - copilotModelEmptyFallbackRegex, - `$1env.COPILOT_MODEL$2` + copilotModelOverrideRegex, + '$1${{ env.COPILOT_MODEL }}' ); expect(result).toBe( - ` COPILOT_MODEL: \${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }}\n` + ` COPILOT_MODEL: \${{ env.COPILOT_MODEL }}\n` ); }); - it('should not modify already-rewritten env.COPILOT_MODEL fallback', () => { + it('should replace repo-level override fallback with workflow-level env.COPILOT_MODEL', () => { const input = " COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }}\n"; const result = input.replace( - copilotModelEmptyFallbackRegex, - `$1env.COPILOT_MODEL$2` + copilotModelOverrideRegex, + '$1${{ env.COPILOT_MODEL }}' + ); + expect(result).toBe(` COPILOT_MODEL: \${{ env.COPILOT_MODEL }}\n`); + }); + + it('should be idempotent when already using workflow-level env.COPILOT_MODEL', () => { + const input = " COPILOT_MODEL: ${{ env.COPILOT_MODEL }}\n"; + const result = input.replace( + copilotModelOverrideRegex, + '$1${{ env.COPILOT_MODEL }}' ); expect(result).toBe(input); }); diff --git a/scripts/ci/postprocess-smoke-workflows.ts b/scripts/ci/postprocess-smoke-workflows.ts index ca5640d5..2d7c702a 100644 --- a/scripts/ci/postprocess-smoke-workflows.ts +++ b/scripts/ci/postprocess-smoke-workflows.ts @@ -110,18 +110,15 @@ const legacyApiProxyLogsDirRegex = // postinstall script downloads the platform-specific native binary. Without it, // `claude` fails with "native binary not installed". -// Work around gh-aw compiler bug (gh-aw#26565) where Copilot model fallback is -// emitted at the step level and overrides the workflow-level COPILOT_MODEL env -// when the repo variables are unset. Older compilers emitted an empty string -// fallback (`|| ''`); newer compilers emit a hardcoded default model -// (`|| 'claude-sonnet-4.6'`) and may add an extra `vars.GH_AW_DEFAULT_MODEL_COPILOT` -// link in the fallback chain. In both cases the step-level value wins over the -// workflow-level `env: COPILOT_MODEL: ...` we set on BYOK smoke workflows, which -// breaks targeted BYOK testing (e.g. forcing `o4-mini-aw` against Azure OpenAI). -// We replace the entire expression with `env.COPILOT_MODEL` so the step inherits -// the workflow-level default verbatim. -const copilotModelEmptyFallbackRegex = - /(COPILOT_MODEL:\s*\$\{\{\s*vars\.GH_AW_MODEL_AGENT_COPILOT\s*\|\|\s*)(?:vars\.GH_AW_DEFAULT_MODEL_COPILOT\s*\|\|\s*)?'[^']*'(\s*\}\})/g; +// Work around gh-aw compiler bug (gh-aw#26565) where Copilot model selection is +// emitted at the step level for BYOK smoke workflows, overriding the +// workflow-level `env: COPILOT_MODEL: ...` that intentionally pins a +// low-cost/provider-specific model (for example `claude-haiku-4.5` or +// `o4-mini-aw`). Normalize every compiled step-level COPILOT_MODEL expression +// back to `${{ env.COPILOT_MODEL }}` so the workflow-level setting wins even if +// repo-level default model variables are configured. +const copilotModelOverrideRegex = + /^(\s*COPILOT_MODEL:\s*)\$\{\{\s*(?:vars\.GH_AW_MODEL_AGENT_COPILOT\s*\|\|\s*)?(?:vars\.GH_AW_DEFAULT_MODEL_COPILOT\s*\|\|\s*)?(?:env\.COPILOT_MODEL|''|'[^']*')\s*\}\}[ \t]*$/gm; // Sentinel used to detect whether the "Copy Copilot session state" step has // already been replaced with the AWF-aware inline script. @@ -542,20 +539,21 @@ for (const workflowPath of workflowPaths) { } } - // For smoke-copilot-byok variants: replace empty model fallbacks with the - // workflow-level COPILOT_MODEL env so the generated step inherits the shared - // default without hardcoding a duplicate model string here. + // For smoke-copilot-byok variants: replace compiled COPILOT_MODEL override + // expressions with the workflow-level COPILOT_MODEL env so the generated + // step inherits the intended BYOK model instead of any repo-level default. const isCopilotByokSmoke = /smoke-copilot-byok[^/]*\.lock\.yml$/.test(workflowPath); if (isCopilotByokSmoke) { - const emptyFallbackMatches = content.match(copilotModelEmptyFallbackRegex); - if (emptyFallbackMatches) { - content = content.replace( - copilotModelEmptyFallbackRegex, - '$1env.COPILOT_MODEL$2' - ); + const rewrittenContent = content.replace( + copilotModelOverrideRegex, + '$1${{ env.COPILOT_MODEL }}' + ); + if (rewrittenContent !== content) { + const rewrittenCount = (content.match(copilotModelOverrideRegex) || []).length; + content = rewrittenContent; modified = true; console.log( - ` Replaced ${emptyFallbackMatches.length} empty COPILOT_MODEL fallback(s) for BYOK smoke` + ` Rewrote ${rewrittenCount} COPILOT_MODEL override(s) to env.COPILOT_MODEL for BYOK smoke` ); } } diff --git a/scripts/ci/smoke-copilot-byok-workflow.test.ts b/scripts/ci/smoke-copilot-byok-workflow.test.ts new file mode 100644 index 00000000..a9ac615d --- /dev/null +++ b/scripts/ci/smoke-copilot-byok-workflow.test.ts @@ -0,0 +1,26 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +const workflowsDir = path.resolve(__dirname, '../../.github/workflows'); + +const byokSourcePath = path.join(workflowsDir, 'smoke-copilot-byok.md'); +const byokLockPaths = [ + path.join(workflowsDir, 'smoke-copilot-byok.lock.yml'), + path.join(workflowsDir, 'smoke-copilot-byok-aoai-apikey.lock.yml'), + path.join(workflowsDir, 'smoke-copilot-byok-aoai-entra.lock.yml'), +]; + +describe('smoke copilot BYOK workflow model selection', () => { + it('pins the direct BYOK source workflow to claude-haiku-4.5', () => { + const source = fs.readFileSync(byokSourcePath, 'utf-8'); + + expect(source).toContain('COPILOT_MODEL: claude-haiku-4.5'); + }); + + it.each(byokLockPaths)('forces workflow-level COPILOT_MODEL in %s', (lockPath) => { + const lock = fs.readFileSync(lockPath, 'utf-8'); + + expect(lock).toContain('COPILOT_MODEL: ${{ env.COPILOT_MODEL }}'); + expect(lock).not.toContain('COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || env.COPILOT_MODEL }}'); + }); +});