From 8ac117e5b796577466272d1b57c8d6e00e5b2b80 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:39:28 +0000 Subject: [PATCH 1/4] Initial plan From df907c66cedadde07b6c901d2abddd0e17628e10 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 31 Jul 2026 21:52:56 +0000 Subject: [PATCH 2/4] Improve failure issue title for engine 429 rate limits Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/handle_agent_failure.cjs | 25 +++++++++++++++++++ .../setup/js/handle_agent_failure.test.cjs | 2 ++ 2 files changed, 27 insertions(+) diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index b1e56e2ee12..de8932ed482 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -291,6 +291,7 @@ function buildFailureMatchCategories(options) { * @param {boolean} options.hasStaleLockFileFailed * @param {boolean} options.hasDailyAICExceeded * @param {boolean} options.aiCreditsRateLimitError + * @param {boolean} options.hasEngineRateLimit429 * @param {boolean} options.maxAICreditsExceeded * @param {boolean} options.hasAssignmentErrors * @param {boolean} options.http400ResponseError @@ -304,6 +305,7 @@ function buildFailureIssueTitle(options) { if (options.hasDailyAICExceeded) return `[aw] ${workflowName} exceeded daily AI credits budget`; if (options.maxAICreditsExceeded) return `[aw] ${workflowName} exceeded max AI credits`; if (options.aiCreditsRateLimitError) return `[aw] ${workflowName} hit AI credits rate limit`; + if (options.hasEngineRateLimit429) return `[aw] ${workflowName} hit engine rate limit (HTTP 429)`; // Missing model pricing is surfaced by the proxy as HTTP 400, so prefer the // specialized title before falling back to the generic transport-level error. if (options.missingModelPricingError) { @@ -2614,6 +2616,27 @@ function detectAWFFirewallStartupFailureFromLog() { } } +/** + * Detect whether the agent failure was caused by engine HTTP 429/rate limiting. + * Checks agent-stdio.log first, then falls back to OTLP mirror payloads. + * @returns {boolean} + */ +function detectEngineRateLimit429Failure() { + const agentOutputFile = process.env.GH_AW_AGENT_OUTPUT; + const stdioLogPath = agentOutputFile ? path.join(path.dirname(agentOutputFile), "agent-stdio.log") : "/tmp/gh-aw/agent-stdio.log"; + try { + if (fs.existsSync(stdioLogPath)) { + const logContent = fs.readFileSync(stdioLogPath, "utf8"); + if (hasEngineRateLimit429Signal(logContent)) { + return true; + } + } + } catch { + // Ignore read errors and continue with OTLP mirror fallback. + } + return hasEngineRateLimit429InOTELMirror(); +} + /** * Extract terminal error messages from agent-stdio.log to surface engine failures. * First tries to match known error patterns (ERROR:, Error:, Fatal:, panic:, Reconnecting...). @@ -3389,6 +3412,7 @@ async function main() { if (hasToolDenialsExceeded) { core.info(`Detected ${toolDenialsExceededEvents.length} guard.tool_denials_exceeded event(s) from Copilot SDK events.jsonl`); } + const hasEngineRateLimit429 = agentConclusion === "failure" && !maxAICreditsExceeded && detectEngineRateLimit429Failure(); // Detect cache-miss misconfiguration: the agent reported a missing_data with reason // "cache_memory_miss" after a cache restore matched. This indicates the prompt @@ -3537,6 +3561,7 @@ async function main() { hasStaleLockFileFailed, hasDailyAICExceeded, aiCreditsRateLimitError, + hasEngineRateLimit429, maxAICreditsExceeded, hasAssignmentErrors, http400ResponseError, diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index d27867e77bb..c11dbee075d 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -97,6 +97,7 @@ describe("handle_agent_failure", () => { hasStaleLockFileFailed: false, hasDailyAICExceeded: false, aiCreditsRateLimitError: false, + hasEngineRateLimit429: false, maxAICreditsExceeded: false, hasAssignmentErrors: false, http400ResponseError: false, @@ -109,6 +110,7 @@ describe("handle_agent_failure", () => { { flag: "hasDailyAICExceeded", expected: "[aw] Test Workflow exceeded daily AI credits budget" }, { flag: "maxAICreditsExceeded", expected: "[aw] Test Workflow exceeded max AI credits" }, { flag: "aiCreditsRateLimitError", expected: "[aw] Test Workflow hit AI credits rate limit" }, + { flag: "hasEngineRateLimit429", expected: "[aw] Test Workflow hit engine rate limit (HTTP 429)" }, { flag: "http400ResponseError", expected: "[aw] Test Workflow hit HTTP 400 bad request" }, { flag: "unknownModelAICredits", expected: "[aw] Test Workflow has unknown model pricing" }, { flag: "hasAppTokenMintingFailed", expected: "[aw] Test Workflow failed to mint GitHub App token" }, From b0e98e1ce3f39cf39c36f9c2889e2df2f1099919 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:21:41 +0000 Subject: [PATCH 3/4] fix: add engine_rate_limit_429 category to buildFailureMatchCategories and guard against completed agent in detectEngineRateLimit429Failure - Add `engine_rate_limit_429` to `buildFailureMatchCategories` so that engine HTTP 429 rate-limit signals produce a distinct deduplication category instead of falling back to `agent_failure`. Pass `hasEngineRateLimit429` at the `buildFailureMatchCategories` call site in `main()`. - Add `terminal_reason: completed` guard in `detectEngineRateLimit429Failure` so that a transient 429 logged earlier in a run that ultimately succeeded does not cause a rate-limit title to be applied to an unrelated post-processing failure. - Export `detectEngineRateLimit429Failure` and add focused unit tests for the new completed-guard and the new `engine_rate_limit_429` category. Addresses review feedback from copilot-pull-request-reviewer (threads on handle_agent_failure.cjs:3564 and handle_agent_failure.cjs:2629-2633). Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/handle_agent_failure.cjs | 9 +++ .../setup/js/handle_agent_failure.test.cjs | 73 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index de8932ed482..9069c3f06bf 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -253,6 +253,7 @@ function buildFailureMatchCategories(options) { if (options.modelNotSupportedError) categories.push("model_not_supported_error"); if (options.http400ResponseError) categories.push("http_400_response_error"); if (options.aiCreditsRateLimitError) categories.push("ai_credits_rate_limit_error"); + if (options.hasEngineRateLimit429) categories.push("engine_rate_limit_429"); if (options.unknownModelAICredits) categories.push("unknown_model_ai_credits"); if (options.missingModelPricingError) categories.push("missing_model_pricing"); if (options.maxAICreditsExceeded) categories.push("max_ai_credits_exceeded"); @@ -2627,6 +2628,12 @@ function detectEngineRateLimit429Failure() { try { if (fs.existsSync(stdioLogPath)) { const logContent = fs.readFileSync(stdioLogPath, "utf8"); + // If the agent completed successfully (terminal_reason: "completed"), the failure + // was caused by something other than the agent itself. Suppress the 429 signal to + // avoid giving a rate-limit title to an unrelated post-processing failure. + if (/"terminal_reason"[ ]?:[ ]?"completed"/.test(logContent)) { + return false; + } if (hasEngineRateLimit429Signal(logContent)) { return true; } @@ -3591,6 +3598,7 @@ async function main() { modelNotSupportedError, http400ResponseError, aiCreditsRateLimitError, + hasEngineRateLimit429, unknownModelAICredits, missingModelPricingError, maxAICreditsExceeded, @@ -4166,6 +4174,7 @@ module.exports = { hasEngineMaxRunsExceededSignal, hasEngineRateLimit429Signal, hasEngineRateLimit429InOTELMirror, + detectEngineRateLimit429Failure, buildEngineMaxRunsExceededContext, buildEngineRateLimit429Context, hasEngineMaxCacheMissesExceededSignal, diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index c11dbee075d..c7b76075339 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -2947,6 +2947,62 @@ describe("handle_agent_failure", () => { }); }); + // ────────────────────────────────────────────────────── + // detectEngineRateLimit429Failure + // ────────────────────────────────────────────────────── + + describe("detectEngineRateLimit429Failure", () => { + let detectEngineRateLimit429Failure; + const fs = require("fs"); + const path = require("path"); + const os = require("os"); + + /** @type {string} */ + let tmpDir; + /** @type {string} */ + let stdioLogPath; + + beforeEach(() => { + vi.resetModules(); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aw-test-detect-429-")); + stdioLogPath = path.join(tmpDir, "agent-stdio.log"); + process.env.GH_AW_AGENT_OUTPUT = path.join(tmpDir, "agent_output.json"); + ({ detectEngineRateLimit429Failure } = require("./handle_agent_failure.cjs")); + }); + + afterEach(() => { + delete process.env.GH_AW_AGENT_OUTPUT; + delete process.env.GH_AW_OTEL_JSONL_PATH; + if (fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("returns true when stdio log contains a 429 rate-limit signal", () => { + fs.writeFileSync(stdioLogPath, "Failed to get response from the AI model; retried 5 times. Last error: CAPIError: 429 429 Sorry, you've exceeded your rate limit.\n"); + expect(detectEngineRateLimit429Failure()).toBe(true); + }); + + it("returns false when stdio log contains terminal_reason: completed even with a 429 signal", () => { + fs.writeFileSync(stdioLogPath, 'Failed to get response; CAPIError: 429 rate limit\n{"type":"result","subtype":"success","terminal_reason":"completed","num_turns":10}\n'); + expect(detectEngineRateLimit429Failure()).toBe(false); + }); + + it("returns false when stdio log contains terminal_reason: completed and no 429 signal", () => { + fs.writeFileSync(stdioLogPath, '{"type":"result","subtype":"success","terminal_reason":"completed","num_turns":5}\n'); + expect(detectEngineRateLimit429Failure()).toBe(false); + }); + + it("returns false when stdio log has no 429 signal and OTLP mirror is absent", () => { + fs.writeFileSync(stdioLogPath, "Agent exited normally.\n"); + expect(detectEngineRateLimit429Failure()).toBe(false); + }); + + it("returns false when log file does not exist and OTLP mirror is absent", () => { + expect(detectEngineRateLimit429Failure()).toBe(false); + }); + }); + // ────────────────────────────────────────────────────── // hasEngineMaxCacheMissesExceededSignal // ────────────────────────────────────────────────────── @@ -5177,6 +5233,23 @@ describe("handle_agent_failure", () => { }); expect(categories).not.toContain("missing_model_pricing"); }); + + it("returns engine_rate_limit_429 category when hasEngineRateLimit429 is true", () => { + const categories = buildFailureMatchCategories({ + agentConclusion: "failure", + hasEngineRateLimit429: true, + }); + expect(categories).toContain("engine_rate_limit_429"); + expect(categories).not.toContain("agent_failure"); + }); + + it("does not return engine_rate_limit_429 category when hasEngineRateLimit429 is false", () => { + const categories = buildFailureMatchCategories({ + agentConclusion: "failure", + hasEngineRateLimit429: false, + }); + expect(categories).not.toContain("engine_rate_limit_429"); + }); }); describe("failure categories filter behavior", () => { From dc5294729e5de46bdffa768bf293eb33e1a1042d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:26:32 +0000 Subject: [PATCH 4/4] fix: exclude aiCreditsRateLimitError from engine-429 guard and add precedence test - Exclude `aiCreditsRateLimitError` from the `hasEngineRateLimit429` guard so that when an AI-credits rate-limit signal already classifies the failure, the engine-429 detector is not run unnecessarily (mirroring the title precedence chain that places AI-credits above engine rate limits). - Add a test case confirming that `aiCreditsRateLimitError` takes precedence over `hasEngineRateLimit429` in `buildFailureIssueTitle` when both flags are true. Addresses github-actions review threads on handle_agent_failure.cjs:3419 and handle_agent_failure.test.cjs:111. Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/handle_agent_failure.cjs | 2 +- actions/setup/js/handle_agent_failure.test.cjs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/actions/setup/js/handle_agent_failure.cjs b/actions/setup/js/handle_agent_failure.cjs index 9069c3f06bf..09e7fde6868 100644 --- a/actions/setup/js/handle_agent_failure.cjs +++ b/actions/setup/js/handle_agent_failure.cjs @@ -3419,7 +3419,7 @@ async function main() { if (hasToolDenialsExceeded) { core.info(`Detected ${toolDenialsExceededEvents.length} guard.tool_denials_exceeded event(s) from Copilot SDK events.jsonl`); } - const hasEngineRateLimit429 = agentConclusion === "failure" && !maxAICreditsExceeded && detectEngineRateLimit429Failure(); + const hasEngineRateLimit429 = agentConclusion === "failure" && !maxAICreditsExceeded && !aiCreditsRateLimitError && detectEngineRateLimit429Failure(); // Detect cache-miss misconfiguration: the agent reported a missing_data with reason // "cache_memory_miss" after a cache restore matched. This indicates the prompt diff --git a/actions/setup/js/handle_agent_failure.test.cjs b/actions/setup/js/handle_agent_failure.test.cjs index c7b76075339..31ec2878f7d 100644 --- a/actions/setup/js/handle_agent_failure.test.cjs +++ b/actions/setup/js/handle_agent_failure.test.cjs @@ -157,6 +157,10 @@ describe("handle_agent_failure", () => { "[aw] Test Workflow has no AI credits pricing for model (claude-opus-5)" ); }); + + it("aiCreditsRateLimitError takes precedence over hasEngineRateLimit429 when both are true", () => { + expect(buildFailureIssueTitle({ ...baseOptions, aiCreditsRateLimitError: true, hasEngineRateLimit429: true })).toBe("[aw] Test Workflow hit AI credits rate limit"); + }); }); describe("detection caution placement in main()", () => {