Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions actions/setup/js/ai_credits_context.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ const BUDGET_EXCEEDED_EVENT = "budget_exceeded";
// The literal error type emitted by the AWF API proxy (HTTP 400) when maxAiCredits is active
// and the requested model is not in the built-in pricing table.
const UNKNOWN_MODEL_AI_CREDITS_TYPE = "unknown_model_ai_credits";
// The literal error type emitted by the AWF API proxy (HTTP 403) when the consecutive cache
// miss counter reaches the apiProxy.maxCacheMisses limit. Engine-agnostic: all engines share
// the same proxy guardrail.
const MAX_CACHE_MISSES_EXCEEDED_EVENT_TYPE = "max_cache_misses_exceeded";
const MAX_AI_CREDITS_EXCEEDED_STDIO_RE = /maximum ai credits exceeded(?:\s*\((\d+(?:\.\d+)?)\s*\/\s*(\d+(?:\.\d+)?)\))?/i;
const DEFAULT_AGENT_STDIO_LOG = "/tmp/gh-aw/agent-stdio.log";
const AGENT_STDIO_LOG_MAX_TAIL = 64 * 1024; // 64 KB — sufficient for any realistic error block
Expand Down Expand Up @@ -425,6 +429,30 @@ function parseUnknownModelAICreditsAndModelFromAuditLog(auditJsonlPathOverride)
);
}

/**
* Detects a `max_cache_misses_exceeded` event from the AWF API proxy event logs.
* The proxy emits this HTTP 403 error when the consecutive cache miss counter reaches
* the configured `apiProxy.maxCacheMisses` limit. Detection is engine-agnostic:
* all agentic engines share the same AWF API proxy guardrail.
* Structured entries emitted by the AWF API proxy look like:
* { "type": "max_cache_misses_exceeded", "consecutive_cache_misses": 6, "max_cache_misses": 5 }
*
* @param {string} [eventLogPathOverride]
* @returns {boolean}
*/
function parseMaxCacheMissesExceededFromEventLog(eventLogPathOverride) {
return iterateJSONLFiles(
resolveUnknownModelAICreditsLogPaths(eventLogPathOverride),
false,
content => content.includes(MAX_CACHE_MISSES_EXCEEDED_EVENT_TYPE),
(acc, entry) => {
if (acc) return true; // already detected, short-circuit
return traverseObjectTree(entry, (_key, value) => value === MAX_CACHE_MISSES_EXCEEDED_EVENT_TYPE) || undefined;
Comment on lines +443 to +450

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in a0f7e24. Added direct parser coverage for parseMaxCacheMissesExceededFromEventLog, including primary and fallback event-log paths plus negative cases.

},
acc => acc
);
}

/**
* Single-pass combined read of the audit log, returning all AI credits fields at once.
* Used by resolveAICreditsFailureState to avoid reading the same file twice.
Expand Down Expand Up @@ -552,5 +580,7 @@ module.exports = {
parseMaxAICreditsExceededFromAuditLog,
parseUnknownModelAICreditsFromAuditLog,
parseUnknownModelAICreditsAndModelFromAuditLog,
parseMaxCacheMissesExceededFromEventLog,
resolveAICreditsFailureState,
MAX_CACHE_MISSES_EXCEEDED_EVENT_TYPE,
};
54 changes: 54 additions & 0 deletions actions/setup/js/ai_credits_context.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,60 @@ describe("ai_credits_context parseUnknownModelAICreditsAndModelFromAuditLog", ()
});
});

describe("ai_credits_context parseMaxCacheMissesExceededFromEventLog", () => {
let tmpDir;
let parseMaxCacheMissesExceededFromEventLog;

beforeEach(async () => {
vi.resetModules();
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aic-cache-misses-test-"));
delete process.env.GH_AW_AGENT_OUTPUT;
const mod = await import("./ai_credits_context.cjs");
const exports = mod.default || mod;
parseMaxCacheMissesExceededFromEventLog = exports.parseMaxCacheMissesExceededFromEventLog;
});

afterEach(() => {
vi.restoreAllMocks();
fs.rmSync(tmpDir, { recursive: true, force: true });
delete process.env.GH_AW_AGENT_OUTPUT;
});

function writeEventLog(lines, filename = "event-logs.jsonl") {
const logDir = path.join(tmpDir, "sandbox", "firewall", "logs", "api-proxy-logs");
fs.mkdirSync(logDir, { recursive: true });
const logPath = path.join(logDir, filename);
fs.writeFileSync(logPath, lines.map(l => JSON.stringify(l)).join("\n") + "\n", "utf8");
process.env.GH_AW_AGENT_OUTPUT = path.join(tmpDir, "output.json");
return logPath;
}

it("detects max_cache_misses_exceeded in event-logs.jsonl", () => {
writeEventLog([{ type: "max_cache_misses_exceeded", consecutive_cache_misses: 6, max_cache_misses: 5 }]);
expect(parseMaxCacheMissesExceededFromEventLog()).toBe(true);
});

it("detects max_cache_misses_exceeded in events.jsonl fallback", () => {
writeEventLog([{ type: "max_cache_misses_exceeded", consecutive_cache_misses: 7, max_cache_misses: 5 }], "events.jsonl");
expect(parseMaxCacheMissesExceededFromEventLog()).toBe(true);
});

it("returns false when no matching event is present", () => {
writeEventLog([{ type: "response", status: 200 }]);
expect(parseMaxCacheMissesExceededFromEventLog()).toBe(false);
});

it("returns false for missing event log", () => {
process.env.GH_AW_AGENT_OUTPUT = path.join(tmpDir, "output.json");
expect(parseMaxCacheMissesExceededFromEventLog("/nonexistent/path/event-logs.jsonl")).toBe(false);
});

it("does not detect other error types", () => {
writeEventLog([{ type: "unknown_model_ai_credits" }]);
expect(parseMaxCacheMissesExceededFromEventLog()).toBe(false);
});
});

describe("ai_credits_context parseMaxAICreditsFromAuditLog", () => {
let tmpDir;
/** @type {(path?: string) => string} */
Expand Down
46 changes: 42 additions & 4 deletions actions/setup/js/detect_agent_errors.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@

const fs = require("fs");
const { MAX_RUNS_EXCEEDED_PATTERNS, isMaxRunsExceededError } = require("./harness_retry_guard.cjs");
const { parseUnknownModelAICreditsAndModelFromAuditLog } = require("./ai_credits_context.cjs");
const { parseUnknownModelAICreditsAndModelFromAuditLog, parseMaxCacheMissesExceededFromEventLog } = require("./ai_credits_context.cjs");

const LOG_FILE = "/tmp/gh-aw/agent-stdio.log";

Expand Down Expand Up @@ -121,6 +121,17 @@ function buildCombinedPattern(patterns) {
// The pooled per-run invocation budget is saturated — retries cannot make progress.
const INVOCATION_CAP_EXCEEDED_PATTERN = buildCombinedPattern(MAX_RUNS_EXCEEDED_PATTERNS);

// Pattern: AWF API proxy consecutive cache miss limit exceeded.
// The AWF API proxy (engine-agnostic) enforces a configurable limit on back-to-back
// requests that miss the prompt cache (apiProxy.maxCacheMisses, default 5). When the
// limit is reached it rejects further requests with HTTP 403 and error type
// "max_cache_misses_exceeded". Two observable forms:
// 1) JSON error type in provider API response: "max_cache_misses_exceeded"
// 2) Human-readable message from SDK wrapper: "Maximum consecutive cache misses exceeded"
// Structured events from the AWF API proxy event log are checked separately via
// parseMaxCacheMissesExceededFromEventLog().
const MAX_CACHE_MISSES_EXCEEDED_PATTERN = /(?:\bmax_cache_misses_exceeded\b|\bmaximum\s+consecutive\s+cache\s+misses\s+exceeded\b)/i;

/**
* Determines if the collected output contains the observed Copilot/CAPI quota exhaustion error.
* @param {string} output - Collected stdout+stderr from the process
Expand All @@ -142,6 +153,17 @@ function isInvocationCapExceededError(output) {
return isMaxRunsExceededError(output);
}

/**
* Determines if the collected output indicates the AWF API proxy cache miss limit is exceeded.
* Checks the agent stdio log for the text-form signal. The structured AWF API proxy event log
* is checked separately in detectErrors() via parseMaxCacheMissesExceededFromEventLog().
* @param {string} output - Collected stdout+stderr from the process
* @returns {boolean}
*/
function isMaxCacheMissesExceededError(output) {
return MAX_CACHE_MISSES_EXCEEDED_PATTERN.test(output);
}

/**
* Normalize model names to a single safe line for GitHub Actions outputs and issue titles.
* @param {string} value
Expand All @@ -164,7 +186,7 @@ function extractMissingModelPricingModelName(logContent) {
/**
* Detect known error patterns in a log string and return detection results.
* @param {string} logContent - Contents of the agent stdio log
* @returns {{ inferenceAccessError: boolean, mcpPolicyError: boolean, agenticEngineTimeout: boolean, modelNotSupportedError: boolean, http400ResponseError: boolean, capiQuotaExceededError: boolean, invocationCapExceeded: boolean, missingModelPricingError: boolean, missingModelPricingModelName: string }}
* @returns {{ inferenceAccessError: boolean, mcpPolicyError: boolean, agenticEngineTimeout: boolean, modelNotSupportedError: boolean, http400ResponseError: boolean, capiQuotaExceededError: boolean, invocationCapExceeded: boolean, maxCacheMissesExceeded: boolean, missingModelPricingError: boolean, missingModelPricingModelName: string }}
*/
function detectErrors(logContent) {
const missingModelPricingModelName = extractMissingModelPricingModelName(logContent);
Expand All @@ -176,14 +198,15 @@ function detectErrors(logContent) {
http400ResponseError: HTTP_400_RESPONSE_ERROR_PATTERN.test(logContent),
capiQuotaExceededError: isCAPIQuotaExceededError(logContent),
invocationCapExceeded: isInvocationCapExceededError(logContent),
maxCacheMissesExceeded: isMaxCacheMissesExceededError(logContent),
missingModelPricingError: missingModelPricingModelName !== "",
missingModelPricingModelName,
};
}

/**
* Build GitHub Actions output lines from detection results.
* @param {{ inferenceAccessError: boolean, mcpPolicyError: boolean, agenticEngineTimeout: boolean, modelNotSupportedError: boolean, http400ResponseError: boolean, capiQuotaExceededError: boolean, invocationCapExceeded: boolean, missingModelPricingError: boolean, missingModelPricingModelName: string }} results
* @param {{ inferenceAccessError: boolean, mcpPolicyError: boolean, agenticEngineTimeout: boolean, modelNotSupportedError: boolean, http400ResponseError: boolean, capiQuotaExceededError: boolean, invocationCapExceeded: boolean, maxCacheMissesExceeded: boolean, missingModelPricingError: boolean, missingModelPricingModelName: string }} results
* @returns {string[]}
*/
function buildOutputLines(results) {
Expand All @@ -196,14 +219,15 @@ function buildOutputLines(results) {
`http_400_response_error=${results.http400ResponseError}`,
`capi_quota_exceeded_error=${effectiveCAPIQuotaExceeded}`,
`invocation_cap_exceeded=${results.invocationCapExceeded}`,
`max_cache_misses_exceeded=${results.maxCacheMissesExceeded}`,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Addressed in a0f7e24. The detect-agent-errors output is now propagated through main job outputs and conclusion env wiring, and the handler also honors the structured event-log signal.

`missing_model_pricing_error=${results.missingModelPricingError}`,
`missing_model_pricing_model_name=${results.missingModelPricingModelName}`,
];
}

/**
* Write GitHub Actions outputs to $GITHUB_OUTPUT.
* @param {{ inferenceAccessError: boolean, mcpPolicyError: boolean, agenticEngineTimeout: boolean, modelNotSupportedError: boolean, http400ResponseError: boolean, capiQuotaExceededError: boolean, invocationCapExceeded: boolean, missingModelPricingError: boolean, missingModelPricingModelName: string }} results
* @param {{ inferenceAccessError: boolean, mcpPolicyError: boolean, agenticEngineTimeout: boolean, modelNotSupportedError: boolean, http400ResponseError: boolean, capiQuotaExceededError: boolean, invocationCapExceeded: boolean, maxCacheMissesExceeded: boolean, missingModelPricingError: boolean, missingModelPricingModelName: string }} results
*/
function writeOutputs(results) {
const outputFile = process.env.GITHUB_OUTPUT;
Expand Down Expand Up @@ -244,8 +268,17 @@ function main() {
process.stderr.write(`[detect-agent-errors] Detected missing model pricing from firewall structured log: model "${auditModelName}" has no AI credits pricing configured\n`);
}

// Also check the AWF API proxy event logs for the `max_cache_misses_exceeded` structured
// event. This covers all engines since the proxy guardrail fires independently of the
// underlying AI engine.
const eventLogCacheMissesExceeded = parseMaxCacheMissesExceededFromEventLog();
if (eventLogCacheMissesExceeded && !stdioResults.maxCacheMissesExceeded) {
process.stderr.write("[detect-agent-errors] Detected max cache misses exceeded from AWF API proxy event log\n");
}

const results = {
...stdioResults,
maxCacheMissesExceeded: stdioResults.maxCacheMissesExceeded || eventLogCacheMissesExceeded,
missingModelPricingError: stdioResults.missingModelPricingError || auditMissingPricing,
missingModelPricingModelName: stdioResults.missingModelPricingModelName || sanitizeModelName(auditModelName),
};
Expand All @@ -271,6 +304,9 @@ function main() {
if (results.invocationCapExceeded) {
process.stderr.write("[detect-agent-errors] Detected invocation cap exhaustion: the pooled per-run LLM invocation budget is fully saturated\n");
}
if (results.maxCacheMissesExceeded) {
process.stderr.write("[detect-agent-errors] Detected max cache misses exceeded: the AWF API proxy consecutive cache miss limit was reached\n");
}
if (results.missingModelPricingError && !auditMissingPricing) {
process.stderr.write(`[detect-agent-errors] Detected missing model pricing: model "${results.missingModelPricingModelName}" has no AI credits pricing configured\n`);
}
Expand All @@ -287,13 +323,15 @@ module.exports = {
extractMissingModelPricingModelName,
isCAPIQuotaExceededError,
isInvocationCapExceededError,
isMaxCacheMissesExceededError,
INFERENCE_ACCESS_ERROR_PATTERN,
MCP_POLICY_BLOCKED_PATTERN,
AGENTIC_ENGINE_TIMEOUT_PATTERN,
MODEL_NOT_SUPPORTED_PATTERN,
HTTP_400_RESPONSE_ERROR_PATTERN,
CAPI_QUOTA_EXCEEDED_PATTERN,
INVOCATION_CAP_EXCEEDED_PATTERN,
MAX_CACHE_MISSES_EXCEEDED_PATTERN,
MISSING_MODEL_PRICING_PATTERN,
buildOutputLines,
};
101 changes: 101 additions & 0 deletions actions/setup/js/detect_agent_errors.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,15 @@ const {
detectErrors,
isCAPIQuotaExceededError,
isInvocationCapExceededError,
isMaxCacheMissesExceededError,
INFERENCE_ACCESS_ERROR_PATTERN,
MCP_POLICY_BLOCKED_PATTERN,
AGENTIC_ENGINE_TIMEOUT_PATTERN,
MODEL_NOT_SUPPORTED_PATTERN,
HTTP_400_RESPONSE_ERROR_PATTERN,
CAPI_QUOTA_EXCEEDED_PATTERN,
INVOCATION_CAP_EXCEEDED_PATTERN,
MAX_CACHE_MISSES_EXCEEDED_PATTERN,
MISSING_MODEL_PRICING_PATTERN,
extractMissingModelPricingModelName,
buildOutputLines,
Expand Down Expand Up @@ -318,6 +320,7 @@ describe("detect_agent_errors.cjs", () => {
expect(result.http400ResponseError).toBe(false);
expect(result.capiQuotaExceededError).toBe(false);
expect(result.invocationCapExceeded).toBe(false);
expect(result.maxCacheMissesExceeded).toBe(false);
expect(result.missingModelPricingError).toBe(false);
expect(result.missingModelPricingModelName).toBe("");
});
Expand Down Expand Up @@ -484,6 +487,7 @@ describe("detect_agent_errors.cjs", () => {
expect(result.http400ResponseError).toBe(false);
expect(result.capiQuotaExceededError).toBe(false);
expect(result.invocationCapExceeded).toBe(false);
expect(result.maxCacheMissesExceeded).toBe(false);
expect(result.missingModelPricingError).toBe(false);
});

Expand Down Expand Up @@ -515,6 +519,67 @@ commentary" has no AI credits pricing`;
expect(result.missingModelPricingError).toBe(false);
expect(result.missingModelPricingModelName).toBe("");
});

it("detects max cache misses exceeded (JSON error type form)", () => {
const result = detectErrors('{"error":{"type":"max_cache_misses_exceeded","message":"Maximum consecutive cache misses exceeded (6 / 5).","consecutive_cache_misses":6,"max_cache_misses":5}}');
expect(result.maxCacheMissesExceeded).toBe(true);
expect(result.inferenceAccessError).toBe(false);
expect(result.invocationCapExceeded).toBe(false);
});

it("detects max cache misses exceeded (human-readable message form)", () => {
const result = detectErrors("Maximum consecutive cache misses exceeded");
expect(result.maxCacheMissesExceeded).toBe(true);
expect(result.inferenceAccessError).toBe(false);
expect(result.invocationCapExceeded).toBe(false);
});

it("detects max cache misses exceeded in a production log line", () => {
const log = '2026-07-30T06:14:50.000Z [ERROR] Error in API request: 403 {"error":{"type":"max_cache_misses_exceeded","message":"Maximum consecutive cache misses exceeded (6 / 5).","consecutive_cache_misses":6,"max_cache_misses":5}}';
const result = detectErrors(log);
expect(result.maxCacheMissesExceeded).toBe(true);
});

it("does not false-positive on unrelated cache miss content", () => {
const result = detectErrors("Cache miss for key: model-output-xyz");
expect(result.maxCacheMissesExceeded).toBe(false);
});
});

describe("MAX_CACHE_MISSES_EXCEEDED_PATTERN", () => {
it("matches max_cache_misses_exceeded error type", () => {
expect(MAX_CACHE_MISSES_EXCEEDED_PATTERN.test('{"type":"max_cache_misses_exceeded"}')).toBe(true);
});

it("matches Maximum consecutive cache misses exceeded message", () => {
expect(MAX_CACHE_MISSES_EXCEEDED_PATTERN.test("Maximum consecutive cache misses exceeded")).toBe(true);
});

it("is case-insensitive", () => {
expect(MAX_CACHE_MISSES_EXCEEDED_PATTERN.test("MAXIMUM CONSECUTIVE CACHE MISSES EXCEEDED")).toBe(true);
});

it("does not match unrelated cache miss content", () => {
expect(MAX_CACHE_MISSES_EXCEEDED_PATTERN.test("Cache miss for key: output")).toBe(false);
});
});

describe("isMaxCacheMissesExceededError", () => {
it("returns false for empty input", () => {
expect(isMaxCacheMissesExceededError("")).toBe(false);
});

it("detects max_cache_misses_exceeded JSON error type", () => {
expect(isMaxCacheMissesExceededError('{"error":{"type":"max_cache_misses_exceeded"}}')).toBe(true);
});

it("detects human-readable message form", () => {
expect(isMaxCacheMissesExceededError("Maximum consecutive cache misses exceeded")).toBe(true);
});

it("returns false for unrelated content", () => {
expect(isMaxCacheMissesExceededError("Some unrelated error message")).toBe(false);
});
});

describe("buildOutputLines", () => {
Expand Down Expand Up @@ -542,6 +607,7 @@ commentary" has no AI credits pricing`;
http400ResponseError: false,
capiQuotaExceededError: false,
invocationCapExceeded: false,
maxCacheMissesExceeded: false,
missingModelPricingError: true,
missingModelPricingModelName: "claude-opus-5",
});
Expand All @@ -559,12 +625,47 @@ commentary" has no AI credits pricing`;
http400ResponseError: false,
capiQuotaExceededError: false,
invocationCapExceeded: false,
maxCacheMissesExceeded: false,
missingModelPricingError: false,
missingModelPricingModelName: "",
});

expect(lines).toContain("missing_model_pricing_error=false");
expect(lines).toContain("missing_model_pricing_model_name=");
});

it("emits max_cache_misses_exceeded=true when detected", () => {
const lines = buildOutputLines({
inferenceAccessError: false,
mcpPolicyError: false,
agenticEngineTimeout: false,
modelNotSupportedError: false,
http400ResponseError: false,
capiQuotaExceededError: false,
invocationCapExceeded: false,
maxCacheMissesExceeded: true,
missingModelPricingError: false,
missingModelPricingModelName: "",
});

expect(lines).toContain("max_cache_misses_exceeded=true");
});

it("emits max_cache_misses_exceeded=false when not detected", () => {
const lines = buildOutputLines({
inferenceAccessError: false,
mcpPolicyError: false,
agenticEngineTimeout: false,
modelNotSupportedError: false,
http400ResponseError: false,
capiQuotaExceededError: false,
invocationCapExceeded: false,
maxCacheMissesExceeded: false,
missingModelPricingError: false,
missingModelPricingModelName: "",
});

expect(lines).toContain("max_cache_misses_exceeded=false");
});
});
});
Loading
Loading