diff --git a/.github/workflows/smoke-copilot-sub-agents.lock.yml b/.github/workflows/smoke-copilot-sub-agents.lock.yml index e13aa9881a2..d2f455295eb 100644 --- a/.github/workflows/smoke-copilot-sub-agents.lock.yml +++ b/.github/workflows/smoke-copilot-sub-agents.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"fb80548385dd467c27ac89dd362abbd4556097fed8958b4b439ce040ee29ef36","body_hash":"ead03f5b55d16621f4474fb57c28643abfea40c6218edbb0ff309b329c4d4498","strict":true,"agent_id":"copilot","agent_model":"gpt-5.3-codex","engine_versions":{"copilot":"1.0.67","copilot-sdk":"1.0.4"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"36ae4feea7121ed52abbeb80b54188e7cbcfe2f4d66363093317359e34a43d81","body_hash":"ead03f5b55d16621f4474fb57c28643abfea40c6218edbb0ff309b329c4d4498","strict":true,"agent_id":"copilot","agent_model":"gpt-5.4","engine_versions":{"copilot":"1.0.67","copilot-sdk":"1.0.4"}} # gh-aw-manifest: {"version":1,"secrets":["GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0","version":"v7.0.0"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.20"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.20"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.20"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.32","digest":"sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.3.32@sha256:63e46b56dfd70895a701b6fc6dd0189e11e2d875f327f1781e81b31848735477"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b","pinned_image":"ghcr.io/github/gh-aw-node@sha256:529d02eb970b1161aa25c593a9c3df57fdfad5a8add328cb3b6eccef66f3183b"},{"image":"ghcr.io/github/github-mcp-server:v1.5.0","digest":"sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4","pinned_image":"ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4"}]} # This file was automatically generated by gh-aw. DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -114,7 +114,7 @@ jobs: env: GH_AW_INFO_ENGINE_ID: "copilot" GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" - GH_AW_INFO_MODEL: "gpt-5.3-codex" + GH_AW_INFO_MODEL: "gpt-5.4" GH_AW_INFO_VERSION: "1.0.67" GH_AW_INFO_AGENT_VERSION: "1.0.67" GH_AW_INFO_WORKFLOW_NAME: "Smoke Copilot Sub Agents" @@ -803,7 +803,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: gpt-5.3-codex + COPILOT_MODEL: gpt-5.4 COPILOT_SDK_URI: http://127.0.0.1:3002 GH_AW_COPILOT_SDK_DRIVER: 1 GH_AW_COPILOT_SDK_SERVER_ARGS: '["--headless","--no-auto-update","--port","3002","--add-dir","/tmp/gh-aw/","--log-level","all","--log-dir","/tmp/gh-aw/sandbox/agent/logs/","--disable-builtin-mcps","--no-ask-user","--allow-all-tools","--allow-all-paths","--no-custom-instructions"]' @@ -1416,7 +1416,7 @@ jobs: COPILOT_AGENT_RUNNER_TYPE: STANDALONE COPILOT_DUMMY_BYOK: dummy-byok-key-for-offline-mode COPILOT_GITHUB_TOKEN: ${{ github.token }} - COPILOT_MODEL: gpt-5.3-codex + COPILOT_MODEL: gpt-5.4 GH_AW_LLM_PROVIDER: github GH_AW_MAX_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_DETECTION_MAX_AI_CREDITS || '400' }} GH_AW_MAX_TURNS: ${{ vars.GH_AW_DEFAULT_MAX_TURNS || '' }} @@ -1512,7 +1512,7 @@ jobs: GH_AW_DETECTION_REASON: ${{ needs.detection.outputs.detection_reason }} GH_AW_EFFECTIVE_TOKENS: ${{ needs.agent.outputs.effective_tokens }} GH_AW_ENGINE_ID: "copilot" - GH_AW_ENGINE_MODEL: "gpt-5.3-codex" + GH_AW_ENGINE_MODEL: "gpt-5.4" GH_AW_ENGINE_VERSION: "1.0.67" GH_AW_PROJECT_UTC: "-08:00" GH_AW_RUNTIME_FEATURES: ${{ vars.GH_AW_RUNTIME_FEATURES }} diff --git a/.github/workflows/smoke-copilot-sub-agents.md b/.github/workflows/smoke-copilot-sub-agents.md index 7ba22229ed5..8e34ec69da3 100644 --- a/.github/workflows/smoke-copilot-sub-agents.md +++ b/.github/workflows/smoke-copilot-sub-agents.md @@ -12,7 +12,7 @@ strict: true engine: id: copilot copilot-sdk: true - model: gpt-5.3-codex + model: gpt-5.4 bare: true safe-outputs: create-issue: diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 144cc16e005..449b12eb23d 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -97,10 +97,10 @@ const HTTP_400_RESPONSE_ERROR_PATTERN = /(?:Response status code does not indica // This is a persistent policy configuration error — retrying will not help. const MCP_POLICY_BLOCKED_PATTERN = /MCP servers were blocked by policy:/; -// Pattern to detect "model not supported" error (e.g. Copilot Pro/Education users hitting -// a model that is unavailable for their subscription tier). +// Pattern to detect "model not supported" / unavailable errors (e.g. Copilot users hitting +// a model that is unavailable for their subscription tier or current policy enablement). // This is a persistent configuration error — retrying with --continue will not help. -const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/; +const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot)/i; // Pattern to detect missing authentication credentials. // On a --continue attempt this may indicate that the Copilot CLI's on-disk session diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 30dcbbbaba7..04dab4eff6e 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -897,7 +897,7 @@ describe("copilot_harness.cjs", () => { describe("MCP policy error prevents retry", () => { // Inline the same retry logic as the driver, including MCP policy check const MCP_POLICY_BLOCKED_PATTERN = /MCP servers were blocked by policy:/; - const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/; + const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot)/i; const MAX_RETRIES = 3; /** @@ -934,6 +934,15 @@ describe("copilot_harness.cjs", () => { expect(shouldRetry(result, 0)).toBe(false); }); + it("does not retry Copilot SDK no-model policy-enablement errors", () => { + const result = { + exitCode: 1, + hasOutput: true, + output: "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot", + }; + expect(shouldRetry(result, 0)).toBe(false); + }); + it("still retries non-policy errors with output", () => { const result = { exitCode: 1, hasOutput: true, output: "CAPIError: 400 Bad Request" }; expect(shouldRetry(result, 0)).toBe(true); @@ -941,13 +950,18 @@ describe("copilot_harness.cjs", () => { }); describe("model-not-supported detection pattern", () => { - const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/; + const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot)/i; it("matches the exact error from the issue report", () => { const errorOutput = "Execution failed: CAPIError: 400 The requested model is not supported."; expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true); }); + it("matches the Copilot SDK no-model policy-enablement error", () => { + const errorOutput = "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot"; + expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true); + }); + describe("copilot output detection + workflow outputs", () => { afterEach(() => { delete process.env.GITHUB_OUTPUT; @@ -992,6 +1006,17 @@ describe("copilot_harness.cjs", () => { expect(content).toContain("model_not_supported_error=false"); expect(content).toContain("http_400_response_error=true"); }); + + it("detects Copilot SDK no-model policy-enablement output", () => { + const output = "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot"; + expect(detectCopilotErrors(output)).toEqual({ + inferenceAccessError: false, + mcpPolicyError: false, + agenticEngineTimeout: false, + modelNotSupportedError: true, + http400ResponseError: false, + }); + }); }); it("matches when embedded in larger log output", () => { diff --git a/actions/setup/js/detect_agent_errors.cjs b/actions/setup/js/detect_agent_errors.cjs index a101ed35e0e..96b5780d8a1 100644 --- a/actions/setup/js/detect_agent_errors.cjs +++ b/actions/setup/js/detect_agent_errors.cjs @@ -55,13 +55,14 @@ const AGENTIC_ENGINE_TIMEOUT_PATTERN = /signal=SIG(?:TERM|KILL|INT)/; // Pattern: Configured model is invalid or unavailable. // Covers common engine/provider variants: // - "The requested model is not supported" +// - "No model available. Check policy enablement under GitHub Settings > Copilot" // - "invalid model name '...'" // - "unknown model " // - "model ... not found" // - "model ... does not exist" // - "Model not found" (standalone, e.g. AIC api-proxy 404: "404 Not Found: Model not found") const MODEL_NOT_SUPPORTED_PATTERN = - /(?:The requested model is not supported|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable)|404\b[^\n]*\bModel\s+not\s+found)/i; + /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable)|404\b[^\n]*\bModel\s+not\s+found)/i; // Pattern: Generic HTTP 400 Bad Request responses emitted by engine / SDK wrappers. // NOTE: keep in sync with HTTP_400_RESPONSE_ERROR_PATTERN in copilot_harness.cjs. diff --git a/actions/setup/js/detect_agent_errors.test.cjs b/actions/setup/js/detect_agent_errors.test.cjs index 2717a8b2927..a54d051e293 100644 --- a/actions/setup/js/detect_agent_errors.test.cjs +++ b/actions/setup/js/detect_agent_errors.test.cjs @@ -106,6 +106,11 @@ describe("detect_agent_errors.cjs", () => { expect(MODEL_NOT_SUPPORTED_PATTERN.test(log)).toBe(true); }); + it("matches Copilot SDK policy-enableable no-model errors", () => { + const errorOutput = "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot"; + expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true); + }); + it("matches invalid/unknown model name variants", () => { expect(MODEL_NOT_SUPPORTED_PATTERN.test("invalid model name 'claude-sonnet-999'")).toBe(true); expect(MODEL_NOT_SUPPORTED_PATTERN.test("unknown model gpt-unknown")).toBe(true); @@ -261,6 +266,16 @@ describe("detect_agent_errors.cjs", () => { expect(result.capiQuotaExceededError).toBe(false); }); + it("detects Copilot SDK no-model policy-enablement errors as model-not-supported", () => { + const result = detectErrors("Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot"); + expect(result.inferenceAccessError).toBe(false); + expect(result.mcpPolicyError).toBe(false); + expect(result.agenticEngineTimeout).toBe(false); + expect(result.modelNotSupportedError).toBe(true); + expect(result.http400ResponseError).toBe(false); + expect(result.capiQuotaExceededError).toBe(false); + }); + it("detects invalid model name errors", () => { const result = detectErrors("Error: invalid model name 'claude-sonnet-999'"); expect(result.inferenceAccessError).toBe(false);