From 68576d55432480c53e17decddcb3a8fe79c986c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Mon, 1 Jun 2026 16:36:56 -0500 Subject: [PATCH] fix: improve maestro test output --- src/__tests__/cli-network.test.ts | 18 +++--- src/cli-test.ts | 22 ++++++-- .../__tests__/session-test-suite.test.ts | 3 + src/daemon/handlers/session-test.ts | 4 ++ src/daemon/request-progress-protocol.ts | 55 +++++++++++++++---- src/daemon/request-progress.ts | 1 + src/utils/__tests__/daemon-client.test.ts | 10 ++-- 7 files changed, 84 insertions(+), 29 deletions(-) diff --git a/src/__tests__/cli-network.test.ts b/src/__tests__/cli-network.test.ts index 5805fce82..d98a47dff 100644 --- a/src/__tests__/cli-network.test.ts +++ b/src/__tests__/cli-network.test.ts @@ -106,7 +106,7 @@ test('test command prints suite summary and exits non-zero on failures', async ( assert.equal(result.calls.length, 1); assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test'); assert.match(result.stderr, /Running replay suite\.\.\./); - assert.doesNotMatch(result.stdout, /PASS \/tmp\/01-pass\.ad/); + assert.match(result.stdout, /PASS 01-pass\.ad \(0\.01s\)/); assert.match( result.stdout, /FAIL "Checkout failure" in 02-fail\.ad after 2 attempts \(total 0\.005s\)/, @@ -204,9 +204,9 @@ test('test command --verbose prints step telemetry for passing tests without deb assert.equal(result.code, null); assert.equal(result.calls[0]?.meta?.debug, false); assert.match(result.stdout, /PASS "Authentication flow" \(0\.5s\)/); - assert.match(result.stdout, /steps \(attempt 1\):/); - assert.match(result.stdout, /\[ok\] tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/); - assert.match(result.stdout, /\[ok\] assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/); + assert.match(result.stdout, /steps:/); + assert.match(result.stdout, /tapOn "text=\\"Log in\\"" \(line 3, 0\.25s\)/); + assert.match(result.stdout, /assertVisible "text=\\"Home\\"" \(line 4, 0\.075s\)/); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); } @@ -248,6 +248,10 @@ test('test command reports flaky passed-on-retry cases in the default summary', assert.equal(result.code, null); assert.match(result.stderr, /Running replay suite\.\.\./); assert.doesNotMatch(result.stdout, /FLAKY/); + assert.match( + result.stdout, + /PASS "Authentication flow" after 2 attempts \(passed attempt 17\.5s, total 112\.2s\)/, + ); assert.match(result.stdout, /Test summary: 1 passed, 0 failed, 1 flaky in 0\.025s/); assert.match(result.stdout, /Flaky tests:/); assert.match( @@ -336,10 +340,7 @@ test('test command prints failed attempt step telemetry when timing trace exists assert.equal(result.code, 1); assert.match(result.stdout, /steps \(attempt 2\):/); - assert.match( - result.stdout, - /\[ok\] open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/, - ); + assert.match(result.stdout, /open "Demo" \(line 3, 0\.125s, timing \{"launchMs":100\}\)/); assert.match( result.stdout, /\[FAIL\] tapOn "text=\\"Pay\\"" \(line 4, 1\.50s, ASSERTION_FAILED\)/, @@ -381,6 +382,7 @@ test('test --maestro forwards Maestro backend and platform for directory suites' assert.deepEqual(result.calls[0]?.positionals, [tmpDir]); assert.equal(result.calls[0]?.flags?.replayBackend, 'maestro'); assert.equal(result.calls[0]?.flags?.platform, 'android'); + assert.equal(result.calls[0]?.meta?.requestProgress, 'replay-test'); assert.match(result.stderr, /Running replay suite\.\.\./); } finally { await fs.rm(tmpDir, { recursive: true, force: true }); diff --git a/src/cli-test.ts b/src/cli-test.ts index 7b54999e0..2ed9c3d73 100644 --- a/src/cli-test.ts +++ b/src/cli-test.ts @@ -37,8 +37,8 @@ function renderReplayTestSummary( renderVerboseTestResult(entry); } } else { - for (const entry of data.failures) { - renderFailedTestResult(entry); + for (const entry of data.tests) { + renderDefaultTestResult(entry); } } @@ -52,6 +52,18 @@ function renderReplayTestSummary( return getReplayTestExitCode(data); } +function renderDefaultTestResult(result: ReplaySuiteTestResult): void { + if (result.status === 'failed') { + renderFailedTestResult(result); + return; + } + if (result.status !== 'passed') return; + + process.stdout.write( + `PASS ${replayTestDisplayName(result)}${formatReplayTestDurationSuffix(result)}\n`, + ); +} + function renderVerboseTestResult(result: ReplaySuiteTestResult): void { if (result.status === 'failed') { renderFailedTestResult(result); @@ -139,7 +151,7 @@ function replayTestStepLines(result: ReplaySuiteTestResult): string[] { if (stops.length === 0) return []; return [ - `steps (attempt ${result.attempts}):`, + result.attempts > 1 ? `steps (attempt ${result.attempts}):` : 'steps:', ...stops.map((stop) => renderReplayStepTrace(stop, starts.get(stop.step))), ]; } @@ -211,8 +223,8 @@ function renderReplayStepTrace( start: ReplayActionStartTrace | undefined, ): string { const failed = stop.ok === false; - const status = failed ? '[FAIL]' : stop.ok === true ? '[ok]' : '[info]'; - return ` ${status} ${formatReplayStepCommand(start, stop)}${formatReplayStepDetails(stop, start)}`; + const status = failed ? '[FAIL] ' : stop.ok === true ? '' : '[info] '; + return ` ${status}${formatReplayStepCommand(start, stop)}${formatReplayStepDetails(stop, start)}`; } function formatReplayStepDetails( diff --git a/src/daemon/handlers/__tests__/session-test-suite.test.ts b/src/daemon/handlers/__tests__/session-test-suite.test.ts index 46e24d972..9fd5a06d1 100644 --- a/src/daemon/handlers/__tests__/session-test-suite.test.ts +++ b/src/daemon/handlers/__tests__/session-test-suite.test.ts @@ -141,16 +141,19 @@ test('test emits progress when attempts retry and pass', async () => { expect(events.map((event) => event.status)).toEqual(['fail', 'pass']); expect(events[0]).toMatchObject({ type: 'replay-test', + title: undefined, status: 'fail', index: 1, total: 1, attempt: 1, maxAttempts: 2, + durationMs: expect.any(Number), retrying: true, message: 'Replay failed at step 1 (open "Demo"): first attempt failed', }); expect(events[1]).toMatchObject({ type: 'replay-test', + title: undefined, status: 'pass', index: 1, total: 1, diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index 39fd2d2e2..7dd3dcf31 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -208,11 +208,13 @@ async function runReplayTestCase( emitRequestProgress({ type: 'replay-test', file: entry.path, + title: entry.title, status: 'fail', index: suiteIndex, total: suiteTotal, attempt: attempts, maxAttempts: retries + 1, + durationMs: finalAttemptDurationMs, retrying: true, message: response.error.message, }); @@ -223,6 +225,7 @@ async function runReplayTestCase( emitRequestProgress({ type: 'replay-test', file: entry.path, + title: entry.title, status: 'pass', index: suiteIndex, total: suiteTotal, @@ -255,6 +258,7 @@ async function runReplayTestCase( emitRequestProgress({ type: 'replay-test', file: entry.path, + title: entry.title, status: 'fail', index: suiteIndex, total: suiteTotal, diff --git a/src/daemon/request-progress-protocol.ts b/src/daemon/request-progress-protocol.ts index ecbe50b2c..307ded613 100644 --- a/src/daemon/request-progress-protocol.ts +++ b/src/daemon/request-progress-protocol.ts @@ -1,3 +1,4 @@ +import path from 'node:path'; import type { DaemonRequest, DaemonResponse } from './types.ts'; import type { RequestProgressEvent } from './request-progress.ts'; @@ -49,19 +50,51 @@ export function serializeDaemonRpcResponseEnvelope(response: unknown): string { export function formatRequestProgressEvent(event: RequestProgressEvent): string | undefined { if (event.type !== 'replay-test') return undefined; - const parts = [event.status, `${event.index}/${event.total}`, event.file]; - if (event.attempt !== undefined && event.maxAttempts !== undefined) { - parts.push(`attempt=${event.attempt}/${event.maxAttempts}`); - } - if (event.retrying) parts.push('retry=true'); - if (event.durationMs !== undefined) - parts.push(`duration=${formatDurationSeconds(event.durationMs)}`); - if (event.artifactsDir && event.status === 'fail') parts.push(`artifacts=${event.artifactsDir}`); + const name = formatReplayTestProgressName(event); + const durationSuffix = + event.durationMs !== undefined ? ` (${formatReplayProgressDuration(event)})` : ''; + const attemptSuffix = formatReplayProgressAttemptSuffix(event); const message = event.message?.replace(/\s+/g, ' ').trim(); - if (message) parts.push(message); - return parts.join(' '); + + if (event.status === 'pass') { + return `PASS ${name}${attemptSuffix}${durationSuffix}`; + } + if (event.status === 'skip') { + return [`SKIP ${name}`, message ? ` ${message}` : ''].filter(Boolean).join('\n'); + } + + return [ + `FAIL ${name}${attemptSuffix}${durationSuffix}`, + message ? ` ${message}` : '', + event.artifactsDir ? ` artifacts: ${event.artifactsDir}` : '', + ] + .filter(Boolean) + .join('\n'); +} + +function formatReplayTestProgressName(event: RequestProgressEvent): string { + const title = event.title?.trim(); + if (title) return JSON.stringify(title); + return path.basename(event.file); +} + +function formatReplayProgressAttemptSuffix(event: RequestProgressEvent): string { + if (event.attempt === undefined) return ''; + if (event.status === 'fail' && event.retrying && event.maxAttempts !== undefined) { + return ` attempt ${event.attempt}/${event.maxAttempts} retrying`; + } + if (event.attempt > 1) return ` after ${event.attempt} attempts`; + return ''; +} + +function formatReplayProgressDuration(event: RequestProgressEvent): string { + const duration = formatDurationSeconds(event.durationMs ?? 0); + return event.attempt && event.attempt > 1 && !event.retrying ? `total ${duration}` : duration; } function formatDurationSeconds(durationMs: number): string { - return `${(Math.max(0, durationMs) / 1000).toFixed(2)}s`; + const seconds = Math.max(0, durationMs) / 1000; + if (seconds >= 10) return `${seconds.toFixed(1)}s`; + if (seconds >= 1) return `${seconds.toFixed(2)}s`; + return `${seconds.toFixed(3).replace(/0+$/, '').replace(/\.$/, '')}s`; } diff --git a/src/daemon/request-progress.ts b/src/daemon/request-progress.ts index 0e32d3d78..8a47b09b3 100644 --- a/src/daemon/request-progress.ts +++ b/src/daemon/request-progress.ts @@ -3,6 +3,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'; export type ReplayTestProgressEvent = { type: 'replay-test'; file: string; + title?: string; status: 'pass' | 'fail' | 'skip'; index: number; total: number; diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 2f0474cb7..a3bff4a3a 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -397,6 +397,7 @@ test('sendToDaemon prints replay test progress before the socket response', asyn event: { type: 'replay-test', file: '/tmp/01-login.ad', + title: 'Login flow', status: 'fail', index: 1, total: 2, @@ -440,10 +441,8 @@ test('sendToDaemon prints replay test progress before the socket response', asyn }); assert.deepEqual(response, { ok: true, data: { via: 'socket' } }); - assert.match( - stderr, - /fail 1\/2 \/tmp\/01-login\.ad attempt=1\/2 retry=true duration=1\.23s first attempt failed/, - ); + assert.match(stderr, /FAIL "Login flow" attempt 1\/2 retrying \(1\.23s\)/); + assert.match(stderr, / first attempt failed/); } finally { (net as unknown as { createConnection: typeof net.createConnection }).createConnection = originalCreateConnection; @@ -498,6 +497,7 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response', event: { type: 'replay-test', file: '/tmp/02-payments.ad', + title: 'Payments flow', status: 'pass', index: 2, total: 3, @@ -543,7 +543,7 @@ test('sendToDaemon prints replay test progress before the HTTP NDJSON response', assert.deepEqual(response, { ok: true, data: { via: 'http-progress' } }); }); assert.deepEqual(seenPaths, ['GET /agent-device/health', 'POST /agent-device/rpc']); - assert.match(stderr, /pass 2\/3 \/tmp\/02-payments\.ad attempt=1\/1 duration=2\.50s/); + assert.match(stderr, /PASS "Payments flow" \(2\.50s\)/); } finally { (http as unknown as { request: typeof http.request }).request = originalHttpRequest; process.stderr.write = originalStderrWrite;