Fail fast on Copilot SDK model-policy errors and repin Smoke Copilot Sub Agents

.github/workflows/smoke-copilot-sub-agents.md

@@ -12,7 +12,7 @@ strict: true
 engine:
   id: copilot
   copilot-sdk: true
-  model: gpt-5.3-codex
+  model: gpt-5.4
   bare: true
 safe-outputs:
   create-issue:

actions/setup/js/copilot_harness.cjs

@@ -97,10 +97,10 @@ const HTTP_400_RESPONSE_ERROR_PATTERN = /(?:Response status code does not indica
 // This is a persistent policy configuration error — retrying will not help.
 const MCP_POLICY_BLOCKED_PATTERN = /MCP servers were blocked by policy:/;
 
-// Pattern to detect "model not supported" error (e.g. Copilot Pro/Education users hitting
-// a model that is unavailable for their subscription tier).
+// Pattern to detect "model not supported" / unavailable errors (e.g. Copilot users hitting
+// a model that is unavailable for their subscription tier or current policy enablement).
 // This is a persistent configuration error — retrying with --continue will not help.
-const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/;
+const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot)/i;
 
 // Pattern to detect missing authentication credentials.
 // On a --continue attempt this may indicate that the Copilot CLI's on-disk session

actions/setup/js/copilot_harness.test.cjs

@@ -897,7 +897,7 @@ describe("copilot_harness.cjs", () => {
   describe("MCP policy error prevents retry", () => {
     // Inline the same retry logic as the driver, including MCP policy check
     const MCP_POLICY_BLOCKED_PATTERN = /MCP servers were blocked by policy:/;
-    const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/;
+    const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot)/i;
     const MAX_RETRIES = 3;
 
     /**
@@ -934,20 +934,34 @@ describe("copilot_harness.cjs", () => {
       expect(shouldRetry(result, 0)).toBe(false);
     });
 
+    it("does not retry Copilot SDK no-model policy-enablement errors", () => {
+      const result = {
+        exitCode: 1,
+        hasOutput: true,
+        output: "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot",
+      };
+      expect(shouldRetry(result, 0)).toBe(false);
+    });
+
     it("still retries non-policy errors with output", () => {
       const result = { exitCode: 1, hasOutput: true, output: "CAPIError: 400 Bad Request" };
       expect(shouldRetry(result, 0)).toBe(true);
     });
   });
 
   describe("model-not-supported detection pattern", () => {
-    const MODEL_NOT_SUPPORTED_PATTERN = /The requested model is not supported/;
+    const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot)/i;
 
     it("matches the exact error from the issue report", () => {
       const errorOutput = "Execution failed: CAPIError: 400 The requested model is not supported.";
       expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true);
     });
 
+    it("matches the Copilot SDK no-model policy-enablement error", () => {
+      const errorOutput = "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot";
+      expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true);
+    });
+
     describe("copilot output detection + workflow outputs", () => {
       afterEach(() => {
         delete process.env.GITHUB_OUTPUT;
@@ -992,6 +1006,17 @@ describe("copilot_harness.cjs", () => {
         expect(content).toContain("model_not_supported_error=false");
         expect(content).toContain("http_400_response_error=true");
       });
+
+      it("detects Copilot SDK no-model policy-enablement output", () => {
+        const output = "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot";
+        expect(detectCopilotErrors(output)).toEqual({
+          inferenceAccessError: false,
+          mcpPolicyError: false,
+          agenticEngineTimeout: false,
+          modelNotSupportedError: true,
+          http400ResponseError: false,
+        });
+      });
     });
 
     it("matches when embedded in larger log output", () => {

actions/setup/js/detect_agent_errors.cjs

@@ -55,13 +55,14 @@ const AGENTIC_ENGINE_TIMEOUT_PATTERN = /signal=SIG(?:TERM|KILL|INT)/;
 // Pattern: Configured model is invalid or unavailable.
 // Covers common engine/provider variants:
 //   - "The requested model is not supported"
+//   - "No model available. Check policy enablement under GitHub Settings > Copilot"
 //   - "invalid model name '...'"
 //   - "unknown model <id>"
 //   - "model ... not found"
 //   - "model ... does not exist"
 //   - "Model not found" (standalone, e.g. AIC api-proxy 404: "404 Not Found: Model not found")
 const MODEL_NOT_SUPPORTED_PATTERN =
-  /(?:The requested model is not supported|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable)|404\b[^\n]*\bModel\s+not\s+found)/i;
+  /(?:The requested model is not supported|No model available\.\s*Check policy enablement under GitHub Settings\s*>\s*Copilot|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable)|404\b[^\n]*\bModel\s+not\s+found)/i;
 
 // Pattern: Generic HTTP 400 Bad Request responses emitted by engine / SDK wrappers.
 // NOTE: keep in sync with HTTP_400_RESPONSE_ERROR_PATTERN in copilot_harness.cjs.

actions/setup/js/detect_agent_errors.test.cjs

@@ -106,6 +106,11 @@ describe("detect_agent_errors.cjs", () => {
       expect(MODEL_NOT_SUPPORTED_PATTERN.test(log)).toBe(true);
     });
 
+    it("matches Copilot SDK policy-enableable no-model errors", () => {
+      const errorOutput = "Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot";
+      expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true);
+    });
+
     it("matches invalid/unknown model name variants", () => {
       expect(MODEL_NOT_SUPPORTED_PATTERN.test("invalid model name 'claude-sonnet-999'")).toBe(true);
       expect(MODEL_NOT_SUPPORTED_PATTERN.test("unknown model gpt-unknown")).toBe(true);
@@ -261,6 +266,16 @@ describe("detect_agent_errors.cjs", () => {
       expect(result.capiQuotaExceededError).toBe(false);
     });
 
+    it("detects Copilot SDK no-model policy-enablement errors as model-not-supported", () => {
+      const result = detectErrors("Execution failed: Error: No model available. Check policy enablement under GitHub Settings > Copilot");
+      expect(result.inferenceAccessError).toBe(false);
+      expect(result.mcpPolicyError).toBe(false);
+      expect(result.agenticEngineTimeout).toBe(false);
+      expect(result.modelNotSupportedError).toBe(true);
+      expect(result.http400ResponseError).toBe(false);
+      expect(result.capiQuotaExceededError).toBe(false);
+    });
+
     it("detects invalid model name errors", () => {
       const result = detectErrors("Error: invalid model name 'claude-sonnet-999'");
       expect(result.inferenceAccessError).toBe(false);