diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index a87b16ee47e..a1a4afb2d30 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -323,16 +323,19 @@ function extractAssistantTextFromJsonlLog(logContent) { } catch { continue; } - // v3 schema: turn_end carries the complete assistant message - if (obj.type === "turn_end" && obj.message && Array.isArray(obj.message.content)) { + // v3 schema: turn_end carries the complete assistant message. + // Claude engine's native stream-json format also emits a top-level + // "assistant" event with the same nested message.content array shape + // (e.g. `{"type":"assistant","message":{"content":[{"type":"text","text":...}]}}`), + // so both are handled identically here. + if ((obj.type === "turn_end" || obj.type === "assistant") && obj.message && Array.isArray(obj.message.content)) { for (const part of obj.message.content) { if (part && typeof part.text === "string") { texts.push(part.text); } } - } - // v1 legacy schema: assistant event carries raw text content - if (obj.type === "assistant" && typeof obj.content === "string" && obj.content) { + // v1 legacy schema: assistant event carries raw text content directly + } else if (obj.type === "assistant" && typeof obj.content === "string" && obj.content) { texts.push(obj.content); } } diff --git a/actions/setup/js/run_evals.test.cjs b/actions/setup/js/run_evals.test.cjs index af20bbd69a4..b33ace0f170 100644 --- a/actions/setup/js/run_evals.test.cjs +++ b/actions/setup/js/run_evals.test.cjs @@ -179,6 +179,45 @@ describe("run_evals.cjs", () => { expect(JSON.parse(line).answer).toBe("YES"); }); + it("parses multiple ID-based answers from Claude engine's native assistant JSONL event", async () => { + // Regression test: Claude's stream-json format emits a top-level "assistant" event + // with a nested message.content array (not the v1 legacy plain-string content, nor + // the v3 turn_end wrapper). Previously this shape was not decoded, so the raw + // un-decoded JSON (with literal "\n" escape sequences) was searched instead, + // breaking \b word-boundary matching for any question ID following an embedded + // newline and causing spurious UNKNOWN answers. + vi.stubEnv( + "GH_AW_EVALS_QUESTIONS", + JSON.stringify([ + { id: "adr-check-performed", question: "Checked?" }, + { id: "action-taken", question: "Action taken?" }, + { id: "decision-justified", question: "Justified?" }, + ]) + ); + vi.stubEnv("GH_AW_EVALS_MODEL", "small"); + vi.stubEnv("GITHUB_RUN_ID", "999"); + + const assistantEvent = JSON.stringify({ + type: "assistant", + message: { + model: "claude-sonnet-4-6", + role: "assistant", + content: [{ type: "text", text: "adr-check-performed: NO\naction-taken: YES\ndecision-justified: YES" }], + }, + }); + fs.writeFileSync(EVALS_LOG_PATH, assistantEvent + "\n", "utf8"); + + await parseMain(); + + const lines = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n"); + const results = Object.fromEntries(lines.map(l => [JSON.parse(l).id, JSON.parse(l).answer])); + expect(results).toEqual({ + "adr-check-performed": "NO", + "action-taken": "YES", + "decision-justified": "YES", + }); + }); + it('keeps missing answers as "UNKNOWN"', async () => { vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); vi.stubEnv("GH_AW_EVALS_MODEL", "small"); @@ -209,6 +248,16 @@ describe("run_evals.cjs", () => { expect(extractAssistantTextFromJsonlLog(log)).toBe("Q1: YES"); }); + it("extracts text from Claude engine's native assistant events (message.content array)", () => { + // Claude's stream-json format emits `{"type":"assistant","message":{"content":[...]}}`, + // the same nested shape as turn_end but under the "assistant" type name. + const log = JSON.stringify({ + type: "assistant", + message: { model: "claude-sonnet-4-6", role: "assistant", content: [{ type: "text", text: "adr-check-performed: NO\naction-taken: YES\ndecision-justified: YES" }] }, + }); + expect(extractAssistantTextFromJsonlLog(log)).toBe("adr-check-performed: NO\naction-taken: YES\ndecision-justified: YES"); + }); + it("joins multiple assistant messages with newlines", () => { const lines = [JSON.stringify({ type: "assistant", content: "Q1: YES" }), JSON.stringify({ type: "assistant", content: "Q2: NO" })].join("\n"); expect(extractAssistantTextFromJsonlLog(lines)).toBe("Q1: YES\nQ2: NO");