diff --git a/src/compat/maestro/__tests__/runtime-flow.test.ts b/src/compat/maestro/__tests__/runtime-flow.test.ts new file mode 100644 index 000000000..96e2fe2c3 --- /dev/null +++ b/src/compat/maestro/__tests__/runtime-flow.test.ts @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { CommandFlags } from '../../../core/dispatch.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../../../daemon/types.ts'; +import { invokeMaestroRunFlowWhen } from '../runtime-flow.ts'; + +test('invokeMaestroRunFlowWhen waits briefly for visible conditions', async () => { + let snapshots = 0; + const invokedActions: SessionAction[] = []; + const batchSteps: CommandFlags['batchSteps'] = [ + { command: 'click', positionals: ['label="Dismiss"'] }, + ]; + + const response = await invokeMaestroRunFlowWhen({ + baseReq: { + token: 't', + session: 's', + flags: { platform: 'android' }, + }, + positionals: ['visible', 'label="Dismiss" || text="Dismiss" || id="Dismiss"'], + batchSteps, + line: 12, + step: 4, + invoke: async (req: DaemonRequest): Promise => { + assert.equal(req.command, 'snapshot'); + snapshots += 1; + return { + ok: true, + data: { + createdAt: Date.now(), + nodes: + snapshots === 1 + ? [] + : [ + { + index: 1, + ref: 'e1', + type: 'android.widget.TextView', + label: 'Dismiss', + rect: { x: 201, y: 2180, width: 138, height: 55 }, + depth: 20, + }, + ], + }, + }; + }, + invokeReplayAction: async ({ action }): Promise => { + invokedActions.push(action); + return { ok: true, data: { clicked: true } }; + }, + }); + + assert.equal(response.ok, true); + assert.equal(snapshots, 2); + assert.deepEqual( + invokedActions.map((action) => [action.command, action.positionals]), + [['click', ['label="Dismiss"']]], + ); + if (response.ok) { + assert.equal(response.data?.ran, 1); + } +}); + +test('invokeMaestroRunFlowWhen keeps notVisible conditions immediate', async () => { + let snapshots = 0; + const response = await invokeMaestroRunFlowWhen({ + baseReq: { + token: 't', + session: 's', + flags: { platform: 'android' }, + }, + positionals: ['notVisible', 'label="Loading" || text="Loading" || id="Loading"'], + batchSteps: [{ command: 'click', positionals: ['label="Continue"'] }], + line: 14, + step: 7, + invoke: async (): Promise => { + snapshots += 1; + return { + ok: true, + data: { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.TextView', + label: 'Loading', + rect: { x: 120, y: 420, width: 160, height: 48 }, + depth: 8, + }, + ], + }, + }; + }, + invokeReplayAction: async (): Promise => { + throw new Error('notVisible should skip while the selector is visible'); + }, + }); + + assert.equal(response.ok, true); + assert.equal(snapshots, 1); + if (response.ok) { + assert.equal(response.data?.skipped, true); + } +}); diff --git a/src/compat/maestro/__tests__/runtime-interactions.test.ts b/src/compat/maestro/__tests__/runtime-interactions.test.ts index 9602e0734..c0d2f43f4 100644 --- a/src/compat/maestro/__tests__/runtime-interactions.test.ts +++ b/src/compat/maestro/__tests__/runtime-interactions.test.ts @@ -34,7 +34,7 @@ test('invokeMaestroTapOn resolves mutating taps from the current raw snapshot', expect(clicks).toEqual([['86', '89']]); }); -test('invokeMaestroSwipeScreen uses a conservative Android content-lane directional swipe', async () => { +test('invokeMaestroSwipeScreen uses an Android content-lane directional swipe', async () => { const swipes: string[][] = []; const response = await invokeMaestroSwipeScreen({ baseReq: { @@ -56,7 +56,7 @@ test('invokeMaestroSwipeScreen uses a conservative Android content-lane directio }); expect(response.ok).toBe(true); - expect(swipes).toEqual([['756', '1521', '324', '1521', '300']]); + expect(swipes).toEqual([['864', '1521', '216', '1521', '300']]); }); function currentBreadcrumbSnapshot(): SnapshotState { diff --git a/src/compat/maestro/__tests__/runtime-targets.test.ts b/src/compat/maestro/__tests__/runtime-targets.test.ts index d5d198f4a..7883e0748 100644 --- a/src/compat/maestro/__tests__/runtime-targets.test.ts +++ b/src/compat/maestro/__tests__/runtime-targets.test.ts @@ -169,6 +169,67 @@ test('resolveMaestroNodeFromSnapshot preserves read order for duplicate matches }); }); +test('resolveMaestroNodeFromSnapshot prefers duplicate text on foreground overlapping screen', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.ScrollView', + label: 'Article, Go back, Show Dialog', + rect: { x: 0, y: 120, width: 1080, height: 1800 }, + depth: 6, + }, + { + index: 2, + ref: 'e2', + type: 'android.widget.Button', + label: 'Show Dialog', + rect: { x: 720, y: 980, width: 280, height: 88 }, + enabled: true, + hittable: true, + depth: 14, + parentIndex: 1, + }, + { + index: 30, + ref: 'e30', + type: 'android.widget.ScrollView', + label: 'NewsFeed, Push NewsFeed, Show Dialog', + rect: { x: 0, y: 120, width: 1080, height: 1800 }, + depth: 6, + }, + { + index: 31, + ref: 'e31', + type: 'android.widget.Button', + label: 'Show Dialog', + rect: { x: 720, y: 1320, width: 280, height: 88 }, + enabled: true, + hittable: true, + depth: 14, + parentIndex: 30, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Show Dialog" || text="Show Dialog" || id="Show Dialog"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + { promoteTapTarget: true }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 31 }), + rect: { x: 720, y: 1320, width: 280, height: 88 }, + }); +}); + test('resolveVisibleMaestroNodeFromSnapshot requires visible text matches to be on screen', () => { const snapshot: SnapshotState = { createdAt: Date.now(), diff --git a/src/compat/maestro/runtime-flow.ts b/src/compat/maestro/runtime-flow.ts index 256b096d9..40f062307 100644 --- a/src/compat/maestro/runtime-flow.ts +++ b/src/compat/maestro/runtime-flow.ts @@ -17,9 +17,15 @@ import { readMaestroSelectorPlatform, resolveVisibleMaestroNodeFromSnapshot, } from './runtime-targets.ts'; +import { sleep } from '../../utils/timeouts.ts'; + +const MAESTRO_RUN_FLOW_WHEN_POLICY = { + visibleTimeoutMs: 3000, + visiblePollMs: 250, +} as const; type MaestroRunFlowWhenCondition = - | { ok: true; mode: string; predicate: string; selector: string } + | { ok: true; mode: string; selector: string } | { ok: false; response: DaemonResponse }; export async function invokeMaestroRunFlowWhen(params: { @@ -80,7 +86,6 @@ function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowW return { ok: true, mode, - predicate: mode === 'visible' ? 'visible' : 'hidden', selector, }; } @@ -92,8 +97,53 @@ async function evaluateMaestroRunFlowWhenCondition( }, condition: Extract, ): Promise<{ ok: true; matched: boolean } | { ok: false; response: DaemonResponse }> { + if (condition.mode === 'visible') { + return await waitForMaestroRunFlowVisibleCondition(params, condition); + } + const response = await captureMaestroRawSnapshot(params); if (!response.ok) return { ok: false, response }; + const result = readMaestroRunFlowVisibleCondition(params, condition.selector, response); + if (!result.ok) { + return { + ok: false, + response: result.response, + }; + } + return { ok: true, matched: !result.matched }; +} + +async function waitForMaestroRunFlowVisibleCondition( + params: { + baseReq: ReplayBaseRequest; + invoke: (req: DaemonRequest) => Promise; + }, + condition: Extract, +): Promise<{ ok: true; matched: boolean } | { ok: false; response: DaemonResponse }> { + // Maestro conditionals commonly guard UI that appears immediately after the + // previous command. Keep this bounded and only for visible; notVisible stays + // a point-in-time condition so optional cleanup blocks do not become waits. + const startedAt = Date.now(); + while (true) { + const response = await captureMaestroRawSnapshot(params); + if (!response.ok) return { ok: false, response }; + const result = readMaestroRunFlowVisibleCondition(params, condition.selector, response); + if (!result.ok) return { ok: false, response: result.response }; + if (result.matched) return { ok: true, matched: true }; + if (Date.now() - startedAt >= MAESTRO_RUN_FLOW_WHEN_POLICY.visibleTimeoutMs) { + return { ok: true, matched: false }; + } + await sleep(MAESTRO_RUN_FLOW_WHEN_POLICY.visiblePollMs); + } +} + +function readMaestroRunFlowVisibleCondition( + params: { + baseReq: ReplayBaseRequest; + }, + selector: string, + response: Extract, +): { ok: true; matched: boolean } | { ok: false; response: DaemonResponse } { const snapshot = readSnapshotState(response.data); if (!snapshot) { return { @@ -101,13 +151,13 @@ async function evaluateMaestroRunFlowWhenCondition( response: errorResponse('COMMAND_FAILED', 'Unable to read snapshot data for runFlow.when.'), }; } - const visible = resolveVisibleMaestroNodeFromSnapshot( + const matched = resolveVisibleMaestroNodeFromSnapshot( snapshot, - condition.selector, + selector, readMaestroSelectorPlatform(params.baseReq.flags), getSnapshotReferenceFrame(snapshot), ).ok; - return { ok: true, matched: condition.mode === 'visible' ? visible : !visible }; + return { ok: true, matched }; } async function invokeMaestroRunFlowWhenSteps( diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index a68e02130..1a9db0142 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -296,7 +296,7 @@ function androidHorizontalDirectionalSwipeX( endX: number, ): [number, number] { if (platform !== 'android') return [startX, endX]; - return startX < endX ? [30, 70] : [70, 30]; + return startX < endX ? [20, 80] : [80, 20]; } function resolvePercentScreenSwipe( diff --git a/src/compat/maestro/runtime-targets.ts b/src/compat/maestro/runtime-targets.ts index b0a366fa3..e0cfa1703 100644 --- a/src/compat/maestro/runtime-targets.ts +++ b/src/compat/maestro/runtime-targets.ts @@ -50,6 +50,11 @@ type ReactNativeOverlayFilterResult = { type SnapshotNodeByIndex = Map; +type MaestroMatchWithScreenContainer = { + candidate: MaestroResolvedSnapshotMatch; + container: SnapshotNode & { rect: Rect }; +}; + export function resolveMaestroNodeFromSnapshot( snapshot: SnapshotState, selector: string, @@ -395,11 +400,11 @@ function selectPreferredMaestroSnapshotMatch( promoteTapTarget: boolean, ): MaestroResolvedSnapshotMatch | null { if (!promoteTapTarget || !visibleTextQuery) { - return selectBestMaestroSnapshotMatch(candidates, visibleTextQuery); + return selectBestMaestroSnapshotMatch(nodes, candidates, visibleTextQuery); } return ( selectLocalizedMaestroVisibleTextMatch(nodes, candidates, visibleTextQuery) ?? - selectBestMaestroSnapshotMatch(candidates, visibleTextQuery) + selectBestMaestroSnapshotMatch(nodes, candidates, visibleTextQuery) ); } @@ -412,11 +417,17 @@ function shouldInferMaestroTabSlot( } function selectBestMaestroSnapshotMatch( + nodes: SnapshotState['nodes'], candidates: MaestroResolvedSnapshotMatch[], visibleTextQuery: string | null, ): MaestroResolvedSnapshotMatch | null { + const foregroundCandidates = preferForegroundContainerDuplicateMatches( + nodes, + candidates, + visibleTextQuery, + ); return ( - candidates.sort((left, right) => + foregroundCandidates.sort((left, right) => compareMaestroSnapshotMatches(left, right, visibleTextQuery), )[0] ?? null ); @@ -461,7 +472,73 @@ function selectLocalizedMaestroVisibleTextMatchFromCandidates( ), ); - return selectBestMaestroSnapshotMatch(localized, query); + return selectBestMaestroSnapshotMatch(nodes, localized, query); +} + +function preferForegroundContainerDuplicateMatches( + nodes: SnapshotState['nodes'], + candidates: MaestroResolvedSnapshotMatch[], + visibleTextQuery: string | null, +): MaestroResolvedSnapshotMatch[] { + if (!visibleTextQuery || candidates.length < 2) return candidates; + const exact = candidates.filter( + (candidate) => maestroVisibleTextMatchRank(candidate.node, visibleTextQuery) === 0, + ); + if (exact.length < 2) return candidates; + + const nodeByIndex = buildSnapshotNodeByIndex(nodes); + const withContainers = exact + .map((candidate) => ({ + candidate, + container: findMaestroScreenContainer(nodes, candidate.node, nodeByIndex), + })) + .filter((entry): entry is MaestroMatchWithScreenContainer => Boolean(entry.container)); + if (withContainers.length < 2 || withContainers.length !== exact.length) return candidates; + + const overlapping = withContainers.filter((entry) => + hasOverlappingScreenContainer(entry, withContainers), + ); + if (overlapping.length < 2) return candidates; + + // UIAutomator reports foreground transparent-stack screens later in the + // hierarchy while preserving both screens. Prefer the later overlapping + // screen only for exact duplicate text, so ordinary duplicate rows keep + // Maestro's read-order behavior. + const foregroundContainerIndex = Math.max(...overlapping.map((entry) => entry.container.index)); + const foreground = overlapping + .filter((entry) => entry.container.index === foregroundContainerIndex) + .map((entry) => entry.candidate); + return foreground.length > 0 ? foreground : candidates; +} + +function hasOverlappingScreenContainer( + entry: MaestroMatchWithScreenContainer, + candidates: MaestroMatchWithScreenContainer[], +): boolean { + return candidates.some( + (other) => + other !== entry && + entry.container.index !== other.container.index && + rectOverlapRatio(entry.container.rect, other.container.rect) >= 0.6, + ); +} + +function findMaestroScreenContainer( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): (SnapshotNode & { rect: Rect }) | null { + return findSnapshotAncestor(nodes, node, nodeByIndex, (ancestor) => { + if (!ancestor.rect) return null; + if (!isMaestroScreenContainerType(ancestor)) return null; + if (ancestor.rect.width < 240 || ancestor.rect.height < 320) return null; + return ancestor as SnapshotNode & { rect: Rect }; + }); +} + +function isMaestroScreenContainerType(node: SnapshotNode): boolean { + const type = normalizeType(node.type ?? ''); + return type === 'scrollview' || type === 'scroll-area' || type === 'list'; } function isLocalizedMaestroVisibleTextCandidate(match: MaestroResolvedSnapshotMatch): boolean { @@ -796,6 +873,15 @@ function rectContains(container: Rect, child: Rect): boolean { ); } +function rectOverlapRatio(a: Rect, b: Rect): number { + const left = Math.max(a.x, b.x); + const right = Math.min(a.x + a.width, b.x + b.width); + const top = Math.max(a.y, b.y); + const bottom = Math.min(a.y + a.height, b.y + b.height); + const overlapArea = Math.max(0, right - left) * Math.max(0, bottom - top); + return overlapArea / Math.max(1, Math.min(rectArea(a), rectArea(b))); +} + function maestroVisibleTextMatchRank(node: SnapshotNode, query: string): number { const values = [node.label, extractNodeText(node), node.identifier, node.value].filter( (value): value is string => Boolean(value), diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index 8196dc708..cb4eaa557 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -1479,7 +1479,7 @@ test('runReplayScriptFile uses Android content lane for Maestro horizontal scree calls.map((call) => [call.command, call.positionals]), [ ['snapshot', []], - ['swipe', ['280', '520', '120', '520', '300']], + ['swipe', ['320', '520', '80', '520', '300']], ['swipe', ['360', '520', '40', '520', '300']], ], ); @@ -1612,7 +1612,7 @@ test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absen assert.equal(response.ok, true); assert.deepEqual( calls.map((call) => [call.command, call.positionals]), - [['snapshot', []]], + Array.from({ length: 13 }, () => ['snapshot', []]), ); }); diff --git a/src/platforms/android/__tests__/adb-executor.test.ts b/src/platforms/android/__tests__/adb-executor.test.ts index 46d761a26..6702260e0 100644 --- a/src/platforms/android/__tests__/adb-executor.test.ts +++ b/src/platforms/android/__tests__/adb-executor.test.ts @@ -31,6 +31,7 @@ import { runCmd, runCmdBackground } from '../../../utils/exec.ts'; const mockRunCmd = vi.mocked(runCmd); const mockRunCmdBackground = vi.mocked(runCmdBackground); +const localAdbExecOptions = { detached: process.platform !== 'win32' }; test('createDeviceAdbExecutor routes local commands through adb with the device serial', async () => { const adb = createDeviceAdbExecutor({ @@ -45,7 +46,11 @@ test('createDeviceAdbExecutor routes local commands through adb with the device assert.deepEqual(result, { stdout: 'ok', stderr: '', exitCode: 0 }); assert.deepEqual(mockRunCmd.mock.calls, [ - ['adb', ['-s', 'emulator-5554', 'shell', 'getprop', 'sys.boot_completed'], { timeoutMs: 1000 }], + [ + 'adb', + ['-s', 'emulator-5554', 'shell', 'getprop', 'sys.boot_completed'], + { timeoutMs: 1000, ...localAdbExecOptions }, + ], ]); }); @@ -72,7 +77,7 @@ test('createDeviceAdbExecutor remains a local adb executor inside provider scope assert.equal(result.stdout, 'ok'); assert.deepEqual(providerCalls, []); assert.deepEqual(mockRunCmd.mock.calls, [ - ['adb', ['-s', 'emulator-5554', 'shell', 'echo', 'local'], undefined], + ['adb', ['-s', 'emulator-5554', 'shell', 'echo', 'local'], localAdbExecOptions], ]); }); @@ -168,12 +173,12 @@ test('createLocalAndroidAdbProvider exposes local pull and install capabilities' [ 'adb', ['-s', 'emulator-5554', 'pull', '/sdcard/video.mp4', '/tmp/video.mp4'], - { allowFailure: true }, + { allowFailure: true, ...localAdbExecOptions }, ], [ 'adb', ['-s', 'emulator-5554', 'install', '-r', '-t', '-d', '-g', '/tmp/app.apk'], - { timeoutMs: 2000 }, + { timeoutMs: 2000, ...localAdbExecOptions }, ], ]); }); diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 236a032db..84bd6ae89 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -34,6 +34,7 @@ const VALID_PNG = Buffer.from( ); const mockRunCmd = vi.mocked(runCmd); const mockSleep = vi.mocked(sleep); +const localAdbExecOptions = { detached: process.platform !== 'win32' }; const device: DeviceInfo = { platform: 'android', @@ -303,7 +304,11 @@ test('dumpUiHierarchy returns streamed XML even when exec-out exits non-zero', a assert.equal(result, xml); assert.equal(mockRunCmd.mock.calls.length, 1); - assert.deepEqual(mockRunCmd.mock.calls[0]?.[2], { allowFailure: true, timeoutMs: 8000 }); + assert.deepEqual(mockRunCmd.mock.calls[0]?.[2], { + allowFailure: true, + timeoutMs: 8000, + ...localAdbExecOptions, + }); }); test('snapshotAndroid uses injected helper artifact before stock uiautomator', async () => { @@ -808,8 +813,12 @@ test('dumpUiHierarchy reads fallback XML when dump exits non-zero', async () => ); assert.equal(result, xml); - assert.deepEqual(dumpCall?.[2], { allowFailure: true, timeoutMs: 8000 }); - assert.equal(catCall?.[2], undefined); + assert.deepEqual(dumpCall?.[2], { + allowFailure: true, + timeoutMs: 8000, + ...localAdbExecOptions, + }); + assert.deepEqual(catCall?.[2], localAdbExecOptions); }); test('dumpUiHierarchy does not read a stale fallback file when dump fails without a path', async () => { diff --git a/src/platforms/android/adb-executor.ts b/src/platforms/android/adb-executor.ts index cef62f1b7..4f423a899 100644 --- a/src/platforms/android/adb-executor.ts +++ b/src/platforms/android/adb-executor.ts @@ -184,7 +184,14 @@ function createSerialAdbExecutor(serial: string): AndroidAdbExecutor { // Local adb execution must escape any active provider scope to avoid routing // tunnel-backed providers back into themselves when they shell out to adb. await withoutCommandExecutorOverride( - async () => await runCmd('adb', ['-s', serial, ...args], options), + async () => + await runCmd('adb', ['-s', serial, ...args], { + ...options, + // Some `adb shell` children can survive killing the adb parent and keep + // requests open past timeout. Give each adb call its own process group + // so timeout/abort cleanup can tear down the whole local command tree. + detached: process.platform !== 'win32', + }), ); }