feat(core): retry all provider errors and surface JSON-RPC error details

packages/core/src/evaluation/orchestrator.ts

@@ -1610,7 +1610,9 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
       });
     } catch (error) {
       lastError = error;
-      if (isTimeoutLike(error) && attempt + 1 < attemptBudget) {
+      if (attempt + 1 < attemptBudget) {
+        const delayMs = retryBackoffMs(attempt);
+        await sleep(delayMs, signal);
         attempt += 1;
         continue;
       }
@@ -2567,7 +2569,7 @@ function buildErrorResult(
   failureReasonCode: string,
   verbose?: boolean,
 ): EvaluationResult {
-  const message = error instanceof Error ? error.message : String(error);
+  const message = extractErrorMessage(error);
 
   let agentRequest: JsonObject | undefined;
   let lmRequest: JsonObject | undefined;
@@ -2711,24 +2713,59 @@ function aggregateEvaluatorTokenUsage(scores?: readonly EvaluatorResult[]): Toke
   };
 }
 
-function isTimeoutLike(error: unknown): boolean {
-  if (!error) {
-    return false;
-  }
-  if (
-    typeof DOMException !== 'undefined' &&
-    error instanceof DOMException &&
-    error.name === 'AbortError'
-  ) {
-    return true;
-  }
+/**
+ * Extract a human-readable message from an error of any shape.
+ *
+ * Handles three cases:
+ * 1. Standard Error instances → error.message
+ * 2. Plain objects with a `message` property (e.g. JSON-RPC error objects
+ *    rejected by @agentclientprotocol/sdk) → obj.message
+ * 3. Everything else → String(error)
+ */
+function extractErrorMessage(error: unknown): string {
   if (error instanceof Error) {
-    const name = error.name?.toLowerCase();
-    const message = error.message?.toLowerCase();
-    return name.includes('timeout') || message.includes('timeout');
+    return error.message;
+  }
+  if (error !== null && typeof error === 'object') {
+    const obj = error as Record<string, unknown>;
+    const parts: string[] = [];
+    if (typeof obj.message === 'string' && obj.message) {
+      parts.push(obj.message);
+    }
+    if (typeof obj.code === 'number') {
+      parts.push(`(code ${obj.code})`);
+    }
+    if (parts.length > 0) {
+      return parts.join(' ');
+    }
+    // Fallback: serialize the object so we never return "[object Object]"
+    try {
+      return JSON.stringify(error);
+    } catch {
+      // circular reference or other serialization failure
+    }
   }
-  const value = String(error).toLowerCase();
-  return value.includes('timeout');
+  return String(error);
+}
+
+/** Exponential backoff: 2^attempt * 1000ms (1s, 2s, 4s, …), capped at 30s. */
+function retryBackoffMs(attempt: number): number {
+  return Math.min(2 ** attempt * 1000, 30_000);
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) return Promise.resolve();
+  return new Promise((resolve) => {
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      'abort',
+      () => {
+        clearTimeout(timer);
+        resolve();
+      },
+      { once: true },
+    );
+  });
 }
 
 function mapChildResults(

packages/core/test/evaluation/orchestrator.test.ts

@@ -223,7 +223,7 @@ describe('runTestCase', () => {
     expect(provider.callIndex).toBe(1);
   });
 
-  it('retries timeout errors up to maxRetries', async () => {
+  it('retries provider errors up to maxRetries', async () => {
     const provider = new SequenceProvider('mock', {
       errors: [new Error('Request timeout')],
       responses: [
@@ -244,6 +244,51 @@ describe('runTestCase', () => {
     expect(result.score).toBeGreaterThan(0);
   });
 
+  it('retries non-timeout provider errors up to maxRetries', async () => {
+    const provider = new SequenceProvider('mock', {
+      errors: [new Error('Provider failure')],
+      responses: [
+        {
+          output: [{ role: 'assistant', content: 'Add structured logging.' }],
+        },
+      ],
+    });
+
+    const result = await runEvalCase({
+      evalCase: baseTestCase,
+      provider,
+      target: baseTarget,
+      evaluators: evaluatorRegistry,
+      maxRetries: 1,
+    });
+
+    expect(result.score).toBeGreaterThan(0);
+  });
+
+  it('applies exponential backoff between retries', async () => {
+    const provider = new SequenceProvider('mock', {
+      errors: [new Error('Transient failure')],
+      responses: [
+        {
+          output: [{ role: 'assistant', content: 'Add structured logging.' }],
+        },
+      ],
+    });
+
+    const startMs = Date.now();
+    await runEvalCase({
+      evalCase: baseTestCase,
+      provider,
+      target: baseTarget,
+      evaluators: evaluatorRegistry,
+      maxRetries: 1,
+    });
+    const elapsedMs = Date.now() - startMs;
+
+    // First retry has 2^0 * 1000 = 1000ms backoff
+    expect(elapsedMs).toBeGreaterThanOrEqual(900);
+  });
+
   it('returns error result on unrecoverable failure', async () => {
     const provider = new SequenceProvider('mock', {
       errors: [new Error('Provider failure')],
@@ -266,6 +311,32 @@ describe('runTestCase', () => {
     expect(result.executionError?.message).toContain('Provider failure');
   });
 
+  it('surfaces JSON-RPC error objects with readable messages', async () => {
+    // Simulates @agentclientprotocol/sdk rejecting with a plain JSON-RPC error object
+    const jsonRpcError = { code: -32600, message: 'Invalid request' };
+    const provider: Provider = {
+      id: 'mock:jsonrpc',
+      kind: 'mock' as const,
+      targetName: 'mock',
+      async invoke(): Promise<ProviderResponse> {
+        throw jsonRpcError;
+      },
+    };
+
+    const result = await runEvalCase({
+      evalCase: baseTestCase,
+      provider,
+      target: baseTarget,
+      evaluators: evaluatorRegistry,
+    });
+
+    expect(result.score).toBe(0);
+    expect(result.executionStatus).toBe('execution_error');
+    expect(result.error).toContain('Invalid request');
+    expect(result.error).toContain('code -32600');
+    expect(result.error).not.toContain('[object Object]');
+  });
+
   it('surfaces provider raw.error as evaluation error', async () => {
     const provider = new SequenceProvider('mock', {
       responses: [