diff --git a/docs/adr/0003-ios-ax-snapshot-failure.md b/docs/adr/0003-ios-ax-snapshot-failure.md new file mode 100644 index 000000000..09b0d44de --- /dev/null +++ b/docs/adr/0003-ios-ax-snapshot-failure.md @@ -0,0 +1,70 @@ +# ADR 0003: iOS AX Snapshot Failure Handling + +## Status + +Accepted + +## Context + +iOS XCTest can fail hierarchy capture with `kAXErrorIllegalArgument` when an accessibility tree is +too deep to serialize. Appium's XCUITest guidance documents the practical depth limit: callers may +raise `snapshotMaxDepth` only up to `62`, and elements at depth `63` or greater cannot be returned by +XCTest. React Native screens are a common source of this shape. + +Before this ADR, Agent Device let that XCTest failure escape as a slow runner command. The daemon +could wait for the command deadline, invalidate or kill the runner session as a transport failure, +and then later commands reported `SESSION_NOT_FOUND`. The app tree may still need flattening, but a +snapshot limitation should not break screenshot, logs, app lifecycle, or direct selector commands in +the same runner session. + +Maestro handles this class of failure in its iOS view hierarchy route by using a depth cap of `60`, +detecting `kAXErrorIllegalArgument`, and retrying from a child/window subtree when the app root +cannot be serialized. + +## Decision + +Agent Device treats iOS AX snapshot serialization failure as a typed snapshot failure, not as a +runner transport failure. + +The runner snapshot path now: + +- caps traversal depth at `60`, with lower user-provided `--depth` values still honored +- catches Swift errors and Objective-C exceptions from `XCUIElement.snapshot()` +- classifies `kAXErrorIllegalArgument` as `IOS_AX_SNAPSHOT_FAILED` +- retries app-root failures from `windows.firstMatch`, first child, and first `.other` subtree +- returns a partial snapshot with a warning when fallback succeeds +- returns `IOS_AX_SNAPSHOT_FAILED` with an app-side flattening hint when fallback fails + +Daemon and CLI output preserve runner warnings and runner error hints. Because the error code is not +`COMMAND_FAILED`, runner-session retry and invalidation policy does not treat this typed failure as a +dead transport. + +Direct iOS selector interaction remains the first path for simple selector clicks, and `find id + click` now probes the runner `querySelector` path before taking a full snapshot. If the +direct probe misses or has a transport fallback condition, the normal snapshot-based find path still +executes. + +## Alternatives Considered + +- Flatten every problematic app screen: still useful when the screen must be fully inspectable, but + it moves a tooling failure mode into each app codebase and does not protect other sessions. +- Copy WebDriverAgent/Appium source generation: too broad for Agent Device. The immediate need is + typed fast failure, partial recovery, and session preservation. +- Copy Maestro's hierarchy implementation wholesale: Maestro builds a different AX model and has + its own swizzled max-depth path. Agent Device keeps its existing snapshot model and adopts only the + small recovery behavior that fits the runner protocol. +- Always return an empty snapshot on AX failure: simple, but ambiguous. Users need to know this is an + iOS AX serialization limit and that app-side flattening may be required. + +## Consequences + +Partial fallback snapshots are explicitly marked `truncated` and include a warning. Selectors may be +less accurate against partial trees, so callers should treat screenshot as visual truth and flatten +the app-side accessibility tree when full inspectability is required. + +`IOS_AX_SNAPSHOT_FAILED` should remain a snapshot-domain error. Do not add it to generic retryable +runner transport errors, and do not invalidate the runner session for it. + +Future improvements can add a dedicated regression fixture for a minimal React Native tree that +reproduces the XCTest depth failure. Until then, TypeScript tests guard warning propagation, typed +error preservation, and direct `find id ... click` routing. diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 0965ab9ce..e43da2cea 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -646,12 +646,25 @@ extension RunnerTests { scope: command.scope, raw: command.raw ?? false ) - if options.raw { + do { + let payload: DataPayload + if options.raw { + payload = try snapshotRaw(app: activeApp, options: options) + } else { + payload = try snapshotFast(app: activeApp, options: options) + } needsPostSnapshotInteractionDelay = true - return Response(ok: true, data: snapshotRaw(app: activeApp, options: options)) + return Response(ok: true, data: payload) + } catch let failure as SnapshotCaptureFailure { + return Response( + ok: false, + error: ErrorPayload( + code: failure.code, + message: failure.message, + hint: failure.hint + ) + ) } - needsPostSnapshotInteractionDelay = true - return Response(ok: true, data: snapshotFast(app: activeApp, options: options)) case .screenshot: let screenshot: XCUIScreenshot #if os(macOS) diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 5e60b4906..51354a974 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -94,6 +94,7 @@ struct DataPayload: Codable { let items: [String]? let nodes: [SnapshotNode]? let truncated: Bool? + let warnings: [String]? let gestureStartUptimeMs: Double? let gestureEndUptimeMs: Double? let x: Double? @@ -115,6 +116,7 @@ struct DataPayload: Codable { items: [String]? = nil, nodes: [SnapshotNode]? = nil, truncated: Bool? = nil, + warnings: [String]? = nil, gestureStartUptimeMs: Double? = nil, gestureEndUptimeMs: Double? = nil, x: Double? = nil, @@ -135,6 +137,7 @@ struct DataPayload: Codable { self.items = items self.nodes = nodes self.truncated = truncated + self.warnings = warnings self.gestureStartUptimeMs = gestureStartUptimeMs self.gestureEndUptimeMs = gestureEndUptimeMs self.x = x @@ -154,10 +157,12 @@ struct DataPayload: Codable { struct ErrorPayload: Codable { let code: String? let message: String + let hint: String? - init(code: String? = nil, message: String) { + init(code: String? = nil, message: String, hint: String? = nil) { self.code = code self.message = message + self.hint = hint } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift index 0418a1080..f9ba98bd2 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift @@ -1,6 +1,10 @@ import XCTest extension RunnerTests { + private static let appleSnapshotMaxDepth = 60 + private static let axSnapshotErrorCode = "IOS_AX_SNAPSHOT_FAILED" + private static let axSnapshotHint = + "This can happen with a React Native deep accessibility tree. Flatten app-side if this screen must be fully inspectable; screenshot, logs, appstate, and open --relaunch can still be used in the same session." private static let collapsedTabCandidateTypes: Set = [ .button, .link, @@ -21,6 +25,8 @@ extension RunnerTests { let flatSnapshots: [XCUIElementSnapshot] let snapshotRanges: [ObjectIdentifier: (Int, Int)] let maxDepth: Int + let partial: Bool + let warnings: [String] } private struct SnapshotEvaluation { @@ -33,6 +39,34 @@ extension RunnerTests { let visible: Bool } + struct SnapshotCaptureFailure: Error { + let code: String + let message: String + let hint: String + } + + private struct SnapshotRootCandidate { + let label: String + let element: XCUIElement + } + + private struct SnapshotRootFailure: Error { + let rootLabel: String + let message: String + + var isAxIllegalArgument: Bool { + RunnerTests.isAxIllegalArgument(message) + } + } + + private struct SnapshotRootCapture { + let queryRoot: XCUIElement + let rootSnapshot: XCUIElementSnapshot + let maxDepth: Int + let partial: Bool + let warnings: [String] + } + // MARK: - Snapshot Entry func elementTypeName(_ type: XCUIElement.ElementType) -> String { @@ -75,12 +109,12 @@ extension RunnerTests { } } - func snapshotFast(app: XCUIApplication, options: SnapshotOptions) -> DataPayload { + func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { if let blocking = blockingSystemAlertSnapshot() { return blocking } - guard let context = makeSnapshotTraversalContext(app: app, options: options) else { + guard let context = try makeSnapshotTraversalContext(app: app, options: options) else { return DataPayload(nodes: [], truncated: false) } @@ -108,6 +142,9 @@ extension RunnerTests { parentIndex: nil ) ) + if context.maxDepth == 0 && !context.rootSnapshot.children.isEmpty { + truncated = true + } if context.maxDepth > 0 { let didTruncateFallback = appendCollapsedTabFallbackNodes( to: &nodes, @@ -130,7 +167,7 @@ extension RunnerTests { truncated = true break } - if let limit = options.depth, depth > limit { continue } + if depth > context.maxDepth { continue } let evaluation = evaluateSnapshot(snapshot, in: context) let include = shouldInclude( @@ -155,6 +192,8 @@ extension RunnerTests { for child in snapshot.children.reversed() { stack.append((child, depth + 1, nextVisibleDepth, currentIndex)) } + } else if !snapshot.children.isEmpty { + truncated = true } if !include || isDuplicate { continue } @@ -183,15 +222,19 @@ extension RunnerTests { } - return DataPayload(nodes: nodes, truncated: truncated) + return DataPayload( + nodes: nodes, + truncated: truncated || context.partial, + warnings: snapshotWarnings(context) + ) } - func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) -> DataPayload { + func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload { if let blocking = blockingSystemAlertSnapshot() { return blocking } - guard let context = makeSnapshotTraversalContext(app: app, options: options) else { + guard let context = try makeSnapshotTraversalContext(app: app, options: options) else { return DataPayload(nodes: [], truncated: false) } @@ -203,7 +246,7 @@ extension RunnerTests { truncated = true return } - if let limit = options.depth, depth > limit { return } + if depth > context.maxDepth { return } let evaluation = evaluateSnapshot(snapshot, in: context) let include = shouldInclude( @@ -229,6 +272,10 @@ extension RunnerTests { } let children = snapshot.children + if depth >= context.maxDepth && !children.isEmpty { + truncated = true + return + } for child in children { walk(child, depth: depth + 1, parentIndex: currentIndex) if truncated { return } @@ -236,7 +283,11 @@ extension RunnerTests { } walk(context.rootSnapshot, depth: 0, parentIndex: nil) - return DataPayload(nodes: nodes, truncated: truncated) + return DataPayload( + nodes: nodes, + truncated: truncated || context.partial, + warnings: snapshotWarnings(context) + ) } func snapshotRect(from frame: CGRect) -> SnapshotRect { @@ -304,28 +355,180 @@ extension RunnerTests { private func makeSnapshotTraversalContext( app: XCUIApplication, options: SnapshotOptions - ) -> SnapshotTraversalContext? { - let viewport = snapshotViewport(app: app) - let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app - - let rootSnapshot: XCUIElementSnapshot - do { - rootSnapshot = try queryRoot.snapshot() - } catch { + ) throws -> SnapshotTraversalContext? { + let viewport = safeSnapshotViewport(app: app) + guard let capture = try captureSnapshotRoot(app: app, options: options) else { return nil } - let (flatSnapshots, snapshotRanges) = flattenedSnapshots(rootSnapshot) + let (flatSnapshots, snapshotRanges) = flattenedSnapshots( + capture.rootSnapshot, + maxDepth: capture.maxDepth + ) return SnapshotTraversalContext( - queryRoot: queryRoot, - rootSnapshot: rootSnapshot, + queryRoot: capture.queryRoot, + rootSnapshot: capture.rootSnapshot, viewport: viewport, flatSnapshots: flatSnapshots, snapshotRanges: snapshotRanges, - maxDepth: options.depth ?? Int.max + maxDepth: capture.maxDepth, + partial: capture.partial, + warnings: capture.warnings ) } + private func captureSnapshotRoot( + app: XCUIApplication, + options: SnapshotOptions + ) throws -> SnapshotRootCapture? { + let requestedDepth = max(options.depth ?? Self.appleSnapshotMaxDepth, 0) + let maxDepth = min(requestedDepth, Self.appleSnapshotMaxDepth) + var warnings: [String] = [] + if requestedDepth > Self.appleSnapshotMaxDepth { + warnings.append( + "iOS XCTest snapshot depth \(requestedDepth) was clamped to \(Self.appleSnapshotMaxDepth) to stay below Apple's depth limit." + ) + } + + let scopedRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } + let queryRoot = scopedRoot ?? app + let primaryLabel = scopedRoot == nil ? "app" : "scope" + switch captureElementSnapshot(queryRoot, label: primaryLabel) { + case .success(let snapshot): + return SnapshotRootCapture( + queryRoot: queryRoot, + rootSnapshot: snapshot, + maxDepth: maxDepth, + partial: false, + warnings: warnings + ) + case .failure(let primaryFailure): + guard primaryFailure.isAxIllegalArgument else { + return nil + } + if scopedRoot != nil { + throw axSnapshotFailure([primaryFailure]) + } + + var failures = [primaryFailure] + for candidate in fallbackSnapshotRootCandidates(app: app) { + switch captureElementSnapshot(candidate.element, label: candidate.label) { + case .success(let snapshot): + var partialWarnings = warnings + partialWarnings.append(axSnapshotPartialWarning(from: primaryFailure, recoveredWith: candidate.label)) + return SnapshotRootCapture( + queryRoot: candidate.element, + rootSnapshot: snapshot, + maxDepth: maxDepth, + partial: true, + warnings: partialWarnings + ) + case .failure(let failure): + failures.append(failure) + } + } + throw axSnapshotFailure(failures) + } + } + + private func captureElementSnapshot( + _ element: XCUIElement, + label: String + ) -> Result { + var rootSnapshot: XCUIElementSnapshot? + var swiftErrorMessage: String? + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + do { + rootSnapshot = try element.snapshot() + } catch { + swiftErrorMessage = describeSnapshotError(error) + } + }) + + if let rootSnapshot { + return .success(rootSnapshot) + } + return .failure( + SnapshotRootFailure( + rootLabel: label, + message: exceptionMessage ?? swiftErrorMessage ?? "snapshot returned no root" + ) + ) + } + + private func fallbackSnapshotRootCandidates(app: XCUIApplication) -> [SnapshotRootCandidate] { + var candidates: [SnapshotRootCandidate] = [] + let window = app.windows.firstMatch + if safeElementExists(window) { + candidates.append(SnapshotRootCandidate(label: "window", element: window)) + } + let firstChild = app.children(matching: .any).firstMatch + if safeElementExists(firstChild) { + candidates.append(SnapshotRootCandidate(label: "first child", element: firstChild)) + } + let firstOther = app.children(matching: .other).firstMatch + if safeElementExists(firstOther) { + candidates.append(SnapshotRootCandidate(label: "first other subtree", element: firstOther)) + } + return candidates + } + + private func safeElementExists(_ element: XCUIElement) -> Bool { + var exists = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + exists = element.exists + }) + if let exceptionMessage { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FALLBACK_EXISTS_IGNORED_EXCEPTION=%@", exceptionMessage) + } + return exists + } + + private func safeSnapshotViewport(app: XCUIApplication) -> CGRect { + var viewport = CGRect.infinite + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + viewport = snapshotViewport(app: app) + }) + if let exceptionMessage { + NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_VIEWPORT_IGNORED_EXCEPTION=%@", exceptionMessage) + } + return viewport + } + + private func describeSnapshotError(_ error: Error) -> String { + let localized = error.localizedDescription + let debug = String(describing: error) + if localized.isEmpty { return debug } + if debug == localized { return localized } + return "\(localized) (\(debug))" + } + + private func axSnapshotFailure(_ failures: [SnapshotRootFailure]) -> SnapshotCaptureFailure { + let roots = failures.map { "\($0.rootLabel): \($0.message)" }.joined(separator: "; ") + return SnapshotCaptureFailure( + code: Self.axSnapshotErrorCode, + message: "iOS XCTest snapshot failed with kAXErrorIllegalArgument. \(roots)", + hint: Self.axSnapshotHint + ) + } + + private func axSnapshotPartialWarning( + from failure: SnapshotRootFailure, + recoveredWith rootLabel: String + ) -> String { + return "iOS XCTest snapshot hit kAXErrorIllegalArgument at \(failure.rootLabel); returned a partial shallow snapshot from \(rootLabel). React Native deep accessibility tree detected; flatten app-side if this screen must be fully inspectable." + } + + private static func isAxIllegalArgument(_ message: String) -> Bool { + let normalized = message.lowercased() + return normalized.contains("kaxerrorillegalargument") + || (normalized.contains("illegal argument") && normalized.contains("snapshot")) + } + + private func snapshotWarnings(_ context: SnapshotTraversalContext) -> [String]? { + return context.warnings.isEmpty ? nil : context.warnings + } + private func evaluateSnapshot( _ snapshot: XCUIElementSnapshot, in context: SnapshotTraversalContext @@ -384,24 +587,27 @@ extension RunnerTests { } private func flattenedSnapshots( - _ root: XCUIElementSnapshot + _ root: XCUIElementSnapshot, + maxDepth: Int ) -> ([XCUIElementSnapshot], [ObjectIdentifier: (Int, Int)]) { var ordered: [XCUIElementSnapshot] = [] var ranges: [ObjectIdentifier: (Int, Int)] = [:] @discardableResult - func visit(_ snapshot: XCUIElementSnapshot) -> Int { + func visit(_ snapshot: XCUIElementSnapshot, depth: Int) -> Int { let start = ordered.count ordered.append(snapshot) var end = start - for child in snapshot.children { - end = max(end, visit(child)) + if depth < maxDepth { + for child in snapshot.children { + end = max(end, visit(child, depth: depth + 1)) + } } ranges[ObjectIdentifier(snapshot)] = (start, end) return end } - _ = visit(root) + _ = visit(root, depth: 0) return (ordered, ranges) } diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index 744adcb01..565caf92a 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -70,7 +70,12 @@ export function createAppleInteractor( if (nodes.length === 0 && device.kind === 'simulator') { throw new AppError('COMMAND_FAILED', 'XCTest snapshot returned 0 nodes on iOS simulator.'); } - return { nodes, truncated: result.truncated ?? false, backend: 'xctest' }; + return { + nodes, + truncated: result.truncated ?? false, + backend: 'xctest', + ...(result.warnings ? { warnings: result.warnings } : {}), + }; }, back: async (mode) => { if (device.target === 'tv') { @@ -129,9 +134,16 @@ export function createAppleInteractor( function readAppleSnapshotResult( result: Record, -): Pick { +): Pick { return { nodes: Array.isArray(result.nodes) ? (result.nodes as RawSnapshotNode[]) : undefined, truncated: typeof result.truncated === 'boolean' ? result.truncated : undefined, + warnings: readStringArray(result.warnings), }; } + +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined; + const strings = value.filter((entry): entry is string => typeof entry === 'string'); + return strings.length > 0 ? strings : undefined; +} diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index adebffd9d..45957b39a 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -16,15 +16,27 @@ vi.mock('../../../core/dispatch.ts', async (importOriginal) => { }; }); +vi.mock('../../../platforms/ios/runner-client.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + runIosRunnerCommand: vi.fn(async () => ({})), + }; +}); + import { dispatchCommand } from '../../../core/dispatch.ts'; +import { runIosRunnerCommand } from '../../../platforms/ios/runner-client.ts'; const mockDispatch = vi.mocked(dispatchCommand); +const mockRunnerCommand = vi.mocked(runIosRunnerCommand); beforeEach(() => { mockDispatch.mockReset(); mockDispatch.mockImplementation(async (_device: unknown, command: string) => { return command === 'snapshot' ? { nodes: [] } : {}; }); + mockRunnerCommand.mockReset(); + mockRunnerCommand.mockResolvedValue({}); }); async function runFindClickScenario(options: { @@ -122,6 +134,89 @@ test('handleFindCommands click returns deterministic metadata across locator var } }); +test('handleFindCommands click tries direct iOS id selector before snapshot', async () => { + mockRunnerCommand.mockResolvedValue({ found: true }); + const session = { + ...makeSession('default'), + appBundleId: 'com.example.app', + }; + const { response, invokeCalls } = await runFindClickScenario({ + session, + positionals: ['id', 'continue-button', 'click'], + invoke: async () => ({ x: 10, y: 20 }), + }); + + expect(response.ok).toBe(true); + if (response.ok) { + expect(response.data).toMatchObject({ + selector: 'id="continue-button"', + locator: 'id', + query: 'continue-button', + x: 10, + y: 20, + }); + } + expect(mockRunnerCommand).toHaveBeenCalledTimes(1); + expect(mockRunnerCommand.mock.calls[0]?.[1]).toMatchObject({ + command: 'querySelector', + selectorKey: 'id', + selectorValue: 'continue-button', + appBundleId: 'com.example.app', + }); + expect(mockDispatch).not.toHaveBeenCalledWith( + expect.anything(), + 'snapshot', + expect.anything(), + expect.anything(), + expect.anything(), + ); + expect(invokeCalls[0]).toMatchObject({ + command: 'click', + positionals: ['id="continue-button"'], + flags: expect.objectContaining({ noRecord: true }), + }); +}); + +test('handleFindCommands click falls back to snapshot when direct iOS id selector misses', async () => { + mockRunnerCommand.mockResolvedValue({ found: false }); + const { response, invokeCalls } = await runFindClickScenario({ + session: { + ...makeSession('default'), + appBundleId: 'com.example.app', + }, + positionals: ['id', 'continue-button', 'click'], + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + identifier: 'continue-button', + hittable: true, + rect: { x: 20, y: 30, width: 100, height: 40 }, + parentIndex: 0, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(mockRunnerCommand).toHaveBeenCalledTimes(1); + expect(mockDispatch).toHaveBeenCalledWith( + expect.anything(), + 'snapshot', + [], + undefined, + expect.anything(), + ); + expect(invokeCalls[0]?.positionals).toEqual(['@e2']); +}); + test('handleFindCommands click prefers on-screen duplicate text matches', async () => { const { response, invokeCalls } = await runFindClickScenario({ positionals: ['Sign in', 'click'], diff --git a/src/daemon/handlers/__tests__/snapshot-handler.test.ts b/src/daemon/handlers/__tests__/snapshot-handler.test.ts index ae1689151..be6c034b1 100644 --- a/src/daemon/handlers/__tests__/snapshot-handler.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-handler.test.ts @@ -350,6 +350,59 @@ test('snapshot surfaces filtered-to-zero Android guidance for interactive snapsh } }); +test('snapshot carries iOS AX partial snapshot warnings from the runner', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-ax-partial'; + sessionStore.set(sessionName, { + ...makeSession(sessionName, iosSimulatorDevice), + appBundleId: 'com.example.app', + }); + + mockDispatch.mockResolvedValue({ + nodes: [ + { + index: 0, + depth: 0, + type: 'Window', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ], + truncated: true, + warnings: [ + 'iOS XCTest snapshot hit kAXErrorIllegalArgument at app; returned a partial shallow snapshot from window. React Native deep accessibility tree detected; flatten app-side if this screen must be fully inspectable.', + ], + }); + + const response = await handleSnapshotCommands({ + req: { + token: 't', + session: sessionName, + command: 'snapshot', + positionals: [], + flags: {}, + }, + sessionName, + logPath: '/tmp/daemon.log', + sessionStore, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + const warnings = response.data?.warnings as string[] | undefined; + expect(response.data?.truncated).toBe(true); + expect(warnings).toEqual([expect.stringContaining('kAXErrorIllegalArgument')]); + expect(warnings?.[0]).toContain('React Native deep accessibility tree'); + expect(warnings?.[0]).toContain('flatten app-side'); + } + expect(mockDispatch).toHaveBeenCalledWith( + expect.anything(), + 'snapshot', + [], + undefined, + expect.anything(), + ); +}); + test('snapshot timeout captures Android screenshot evidence with overlay refs', async () => { const sessionName = 'android-timeout-evidence'; const sessionStore = makeAndroidTimeoutEvidenceSession(sessionName); diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 7ad36804b..574720034 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -2,6 +2,7 @@ import { dispatchCommand, resolveTargetDevice } from '../../core/dispatch.ts'; import { sleep } from '../../utils/timeouts.ts'; import { findBestMatchesByLocator, parseFindArgs, type FindLocator } from '../../utils/finders.ts'; import { centerOfRect, type SnapshotState } from '../../utils/snapshot.ts'; +import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts'; import type { DaemonRequest, DaemonResponse } from '../types.ts'; import { SessionStore } from '../session-store.ts'; import { contextFromFlags } from '../context.ts'; @@ -18,6 +19,11 @@ import { errorResponse } from './response.ts'; import { getActiveAndroidSnapshotFreshness } from '../android-snapshot-freshness.ts'; import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts'; import { PUBLIC_COMMANDS } from '../../command-catalog.ts'; +import { + isDirectIosSelectorFallbackError, + type DirectIosSelectorTarget, +} from '../direct-ios-selector.ts'; +import { normalizeError } from '../../utils/errors.ts'; export { parseFindArgs } from '../../utils/finders.ts'; @@ -146,6 +152,9 @@ export async function handleFindCommands(params: { return handleFindWait(ctx, fetchNodes, locator, query, timeoutMs); } + const directIosResponse = await handleDirectIosFindAction(ctx, action); + if (directIosResponse) return directIosResponse; + const { nodes } = await fetchNodes(); const matchResult = resolveFindMatch({ nodes, @@ -409,6 +418,94 @@ async function handleFindClick(ctx: FindContext, match: ResolvedMatch): Promise< return { ok: true, data: matchData }; } +async function handleDirectIosFindAction( + ctx: FindContext, + action: string, +): Promise { + if (action !== 'click') return null; + if (!ctx.session || ctx.session.device.platform !== 'ios') return null; + if (ctx.req.flags?.findFirst || ctx.req.flags?.findLast) return null; + const selector = directIosSelectorForFind(ctx.locator, ctx.query); + if (!selector) return null; + + const query = await queryDirectIosFindSelector(ctx, selector); + if (!query) return null; + if (!query.ok) return query.response; + if (!query.found) return null; + + const response = await ctx.invoke({ + token: ctx.req.token, + session: ctx.sessionName, + command: 'click', + positionals: [selector.raw], + flags: { ...(ctx.req.flags ?? {}), noRecord: true }, + }); + if (!response.ok) return response; + + const responseData = response.data && typeof response.data === 'object' ? response.data : {}; + const data: Record = { + ...responseData, + selector: selector.raw, + locator: ctx.locator, + query: ctx.query, + }; + ctx.sessionStore.recordAction(ctx.session, { + command: ctx.command, + positionals: ctx.req.positionals ?? [], + flags: ctx.req.flags ?? {}, + result: { selector: selector.raw, action: 'click', locator: ctx.locator, query: ctx.query }, + }); + return { ok: true, data }; +} + +async function queryDirectIosFindSelector( + ctx: FindContext, + selector: DirectIosSelectorTarget, +): Promise<{ ok: true; found: boolean } | { ok: false; response: DaemonResponse } | null> { + if (!ctx.session) return null; + try { + const result = await runIosRunnerCommand( + ctx.session.device, + { + command: 'querySelector', + selectorKey: selector.key, + selectorValue: selector.value, + appBundleId: ctx.session.appBundleId, + }, + { + verbose: Boolean(ctx.req.flags?.verbose), + logPath: ctx.logPath, + traceLogPath: ctx.session.trace?.outPath, + requestId: ctx.req.meta?.requestId, + }, + ); + return { ok: true, found: result.found === true }; + } catch (error) { + if (isDirectIosSelectorFallbackError(error)) return null; + return { ok: false, response: { ok: false, error: normalizeError(error) } }; + } +} + +function directIosSelectorForFind( + locator: FindLocator, + query: string, +): DirectIosSelectorTarget | null { + if (locator !== 'id' && locator !== 'label' && locator !== 'text' && locator !== 'value') { + return null; + } + const value = query.trim(); + if (!value) return null; + return { + key: locator, + value, + raw: `${locator}="${escapeSelectorValue(value)}"`, + }; +} + +function escapeSelectorValue(value: string): string { + return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); +} + async function handleFindFill( ctx: FindContext, match: ResolvedMatch, diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 17b5917ec..271125d02 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -49,6 +49,7 @@ type SnapshotData = { backend?: SnapshotBackend; analysis?: AndroidSnapshotAnalysis; androidSnapshot?: AndroidSnapshotBackendMetadata; + warnings?: string[]; }; type AndroidFreshnessReason = 'empty-interactive' | 'sharp-drop' | 'stuck-route'; @@ -59,6 +60,7 @@ export async function captureSnapshot(params: CaptureSnapshotParams): Promise<{ analysis?: AndroidSnapshotAnalysis; androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: AndroidFreshnessCaptureMeta; + warnings?: string[]; }> { if ( (params.device.platform === 'ios' || params.device.platform === 'android') && @@ -81,6 +83,7 @@ export async function captureSnapshot(params: CaptureSnapshotParams): Promise<{ snapshot: buildSnapshotState(data, resolveSnapshotStateFlags(params)), analysis: data.analysis, androidSnapshot: data.androidSnapshot, + warnings: data.warnings, }; } @@ -125,6 +128,7 @@ async function captureAndroidFreshnessAwareSnapshot( analysis?: AndroidSnapshotAnalysis; androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: AndroidFreshnessCaptureMeta; + warnings?: string[]; }> { let latest = await captureSnapshotAttempt(params); let suspiciousReason = getAndroidFreshnessReason(latest, freshness, params); @@ -149,6 +153,7 @@ async function captureAndroidFreshnessAwareSnapshot( snapshot: latest.snapshot, analysis: latest.data.analysis, androidSnapshot: latest.data.androidSnapshot, + warnings: latest.data.warnings, freshness: retryCount > 0 || Boolean(suspiciousReason) ? { diff --git a/src/daemon/snapshot-runtime.ts b/src/daemon/snapshot-runtime.ts index bd6cdcd68..00d6c3e87 100644 --- a/src/daemon/snapshot-runtime.ts +++ b/src/daemon/snapshot-runtime.ts @@ -274,6 +274,7 @@ function createDaemonSnapshotBackend(params: { analysis: capture.analysis, androidSnapshot: capture.androidSnapshot, freshness: capture.freshness, + warnings: capture.warnings, appName: session?.appBundleId ? (session.appName ?? session.appBundleId) : undefined, appBundleId: session?.appBundleId, }; diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts index df1e91d6d..96e94779b 100644 --- a/src/platforms/ios/__tests__/runner-client.test.ts +++ b/src/platforms/ios/__tests__/runner-client.test.ts @@ -585,6 +585,37 @@ test('parseRunnerResponse preserves runner unsupported-operation codes', async ( ); }); +test('parseRunnerResponse preserves iOS AX snapshot failure code and hint', async () => { + const response = new Response( + JSON.stringify({ + ok: false, + error: { + code: 'IOS_AX_SNAPSHOT_FAILED', + message: 'iOS XCTest snapshot failed with kAXErrorIllegalArgument.', + hint: 'Flatten app-side if this screen must be fully inspectable.', + }, + }), + ); + const session = { + ready: true, + } as any; + + await assert.rejects( + () => parseRunnerResponse(response, session, '/tmp/runner.log'), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'IOS_AX_SNAPSHOT_FAILED'); + assert.match(error.message, /kAXErrorIllegalArgument/); + assert.equal( + error.details?.hint, + 'Flatten app-side if this screen must be fully inspectable.', + ); + assert.equal(isRetryableRunnerError(error), false); + return true; + }, + ); +}); + test('isRetryableRunnerError does not retry xcodebuild early-exit errors', () => { const err = new AppError( 'COMMAND_FAILED', diff --git a/src/platforms/ios/runner-session.ts b/src/platforms/ios/runner-session.ts index 4eeba57e3..bff0934f8 100644 --- a/src/platforms/ios/runner-session.ts +++ b/src/platforms/ios/runner-session.ts @@ -492,7 +492,7 @@ export async function executeRunnerCommandWithSession( type RunnerResponsePayload = { ok?: unknown; - error?: { code?: unknown; message?: unknown }; + error?: { code?: unknown; message?: unknown; hint?: unknown }; data?: unknown; }; @@ -516,6 +516,7 @@ export async function parseRunnerResponse( ? toAppErrorCode(rawCode) : 'COMMAND_FAILED'; const errorMessage = typeof json.error?.message === 'string' ? json.error.message : undefined; + const hint = typeof json.error?.hint === 'string' ? json.error.hint : undefined; throw new AppError(errorCode, errorMessage ?? 'Runner error', { runner: json, xcodebuild: { @@ -523,6 +524,7 @@ export async function parseRunnerResponse( stdout: '', stderr: '', }, + hint, logPath, }); }