diff --git a/apps/cli/ai/eval-runner.ts b/apps/cli/ai/eval-runner.ts
index 28f099ecd5..bf7d33ed66 100644
--- a/apps/cli/ai/eval-runner.ts
+++ b/apps/cli/ai/eval-runner.ts
@@ -178,6 +178,8 @@ async function runEval( input: EvalRunnerInput ) {
 	let turnIndex = 0;
 	let numTurns: number | null = null;
 	let success = false;
+	let error: string | null = null;
+	let timedOut = false;
 
 	phaseStartedAt = Date.now();
 	const query = startAiAgent( {
@@ -189,7 +191,10 @@ async function runEval( input: EvalRunnerInput ) {
 	const queryStartedAt = Date.now();
 	let turnStart = queryStartedAt;
 
-	const timeout = setTimeout( () => void query.interrupt(), input.timeoutMs ?? 300000 );
+	const timeout = setTimeout( () => {
+		timedOut = true;
+		void query.interrupt();
+	}, input.timeoutMs ?? 300000 );
 
 	try {
 		for await ( const message of query ) {
@@ -250,6 +255,8 @@ async function runEval( input: EvalRunnerInput ) {
 				numTurns = message.num_turns ?? null;
 			}
 		}
+	} catch ( caught ) {
+		error = caught instanceof Error ? caught.message : String( caught );
 	} finally {
 		clearTimeout( timeout );
 	}
@@ -257,6 +264,8 @@ async function runEval( input: EvalRunnerInput ) {
 
 	return {
 		success,
+		error,
+		timedOut,
 		numTurns,
 		phaseTimingsMs,
 		turnDurationsMs,