From cf400d1ff26baa6a32f541038799c7ce06f4b563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 22 Mar 2026 19:34:45 +0100 Subject: [PATCH 1/2] refactor: split interaction command handlers --- src/daemon/handlers/interaction-common.ts | 18 + src/daemon/handlers/interaction-fill.ts | 182 ++++ src/daemon/handlers/interaction-flags.ts | 32 + src/daemon/handlers/interaction-get.ts | 118 +++ src/daemon/handlers/interaction-is.ts | 151 +++ src/daemon/handlers/interaction-press.ts | 180 ++++ src/daemon/handlers/interaction-scroll.ts | 112 +++ src/daemon/handlers/interaction-selector.ts | 78 ++ src/daemon/handlers/interaction-snapshot.ts | 41 + src/daemon/handlers/interaction-targeting.ts | 84 ++ src/daemon/handlers/interaction.ts | 943 +------------------ 11 files changed, 1028 insertions(+), 911 deletions(-) create mode 100644 src/daemon/handlers/interaction-common.ts create mode 100644 src/daemon/handlers/interaction-fill.ts create mode 100644 src/daemon/handlers/interaction-flags.ts create mode 100644 src/daemon/handlers/interaction-get.ts create mode 100644 src/daemon/handlers/interaction-is.ts create mode 100644 src/daemon/handlers/interaction-press.ts create mode 100644 src/daemon/handlers/interaction-scroll.ts create mode 100644 src/daemon/handlers/interaction-selector.ts create mode 100644 src/daemon/handlers/interaction-snapshot.ts create mode 100644 src/daemon/handlers/interaction-targeting.ts diff --git a/src/daemon/handlers/interaction-common.ts b/src/daemon/handlers/interaction-common.ts new file mode 100644 index 000000000..b5860cbcc --- /dev/null +++ b/src/daemon/handlers/interaction-common.ts @@ -0,0 +1,18 @@ +import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts'; +import type { DaemonCommandContext } from '../context.ts'; +import type { DaemonRequest } from '../types.ts'; +import { SessionStore } from '../session-store.ts'; + +export type ContextFromFlags = ( + flags: CommandFlags | undefined, + appBundleId?: string, + traceLogPath?: string, +) => DaemonCommandContext; + +export type InteractionHandlerParams = { + req: DaemonRequest; + sessionName: string; + sessionStore: SessionStore; + contextFromFlags: ContextFromFlags; + dispatch: typeof dispatchCommand; +}; diff --git a/src/daemon/handlers/interaction-fill.ts b/src/daemon/handlers/interaction-fill.ts new file mode 100644 index 000000000..da94f360f --- /dev/null +++ b/src/daemon/handlers/interaction-fill.ts @@ -0,0 +1,182 @@ +import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { centerOfRect } from '../../utils/snapshot.ts'; +import { buildSelectorChainForNode, splitSelectorFromArgs } from '../selectors.ts'; +import { isFillableType, resolveRefLabel } from '../snapshot-processing.ts'; +import type { DaemonResponse } from '../types.ts'; +import type { InteractionHandlerParams } from './interaction-common.ts'; +import { refSnapshotFlagGuardResponse } from './interaction-flags.ts'; +import { resolveRefTarget } from './interaction-targeting.ts'; +import { resolveSelectorTarget } from './interaction-selector.ts'; + +export async function handleFillCommand(params: InteractionHandlerParams): Promise { + const { req, sessionName, sessionStore, contextFromFlags, dispatch } = params; + const session = sessionStore.get(sessionName); + if (session && !isCommandSupportedOnDevice('fill', session.device)) { + return { + ok: false, + error: { code: 'UNSUPPORTED_OPERATION', message: 'fill is not supported on this device' }, + }; + } + if (req.positionals?.[0]?.startsWith('@')) { + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('fill', req.flags); + if (invalidRefFlagsResponse) return invalidRefFlagsResponse; + const labelCandidate = req.positionals.length >= 3 ? req.positionals[1] : ''; + const text = + req.positionals.length >= 3 + ? req.positionals.slice(2).join(' ') + : req.positionals.slice(1).join(' '); + if (!text) { + return { + ok: false, + error: { code: 'INVALID_ARGS', message: 'fill requires text after ref' }, + }; + } + const resolvedRefTarget = resolveRefTarget({ + session, + refInput: req.positionals[0], + fallbackLabel: labelCandidate, + requireRect: true, + invalidRefMessage: 'fill requires a ref like @e2', + notFoundMessage: `Ref ${req.positionals[0]} not found or has no bounds`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node, snapshotNodes } = resolvedRefTarget.target; + if (!node.rect) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `Ref ${req.positionals[0]} not found or has no bounds`, + }, + }; + } + const nodeType = node.type ?? ''; + const fillWarning = + nodeType && !isFillableType(nodeType, session.device.platform) + ? `fill target ${req.positionals[0]} resolved to "${nodeType}", attempting fill anyway.` + : undefined; + const refLabel = resolveRefLabel(node, snapshotNodes); + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { + action: 'fill', + }); + const { x, y } = centerOfRect(node.rect); + const data = await dispatch( + session.device, + 'fill', + [String(x), String(y), text], + req.flags?.out, + { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + }, + ); + const resultPayload: Record = { + ...(data ?? { ref, x, y }), + }; + if (fillWarning) { + resultPayload.warning = fillWarning; + } + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { ...resultPayload, refLabel, selectorChain }, + }); + return { ok: true, data: resultPayload }; + } + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + const selectorArgs = splitSelectorFromArgs(req.positionals ?? [], { + preferTrailingValue: true, + }); + if (!selectorArgs) { + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: 'fill requires x y text, @ref text, or selector text', + }, + }; + } + if (selectorArgs.rest.length === 0) { + return { + ok: false, + error: { code: 'INVALID_ARGS', message: 'fill requires text after selector' }, + }; + } + const text = selectorArgs.rest.join(' ').trim(); + if (!text) { + return { + ok: false, + error: { code: 'INVALID_ARGS', message: 'fill requires text after selector' }, + }; + } + const resolvedSelectorTarget = await resolveSelectorTarget({ + command: req.command, + selectorExpression: selectorArgs.selectorExpression, + session, + flags: req.flags, + sessionStore, + contextFromFlags, + interactiveOnly: true, + requireRect: true, + requireUnique: true, + disambiguateAmbiguous: true, + dispatch, + }); + if (!resolvedSelectorTarget.ok) return resolvedSelectorTarget.response; + const { resolved, snapshot } = resolvedSelectorTarget; + const node = resolved.node; + if (!node.rect) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `Selector ${resolved.selector.raw} resolved to invalid bounds`, + }, + }; + } + const nodeType = node.type ?? ''; + const fillWarning = + nodeType && !isFillableType(nodeType, session.device.platform) + ? `fill target ${resolved.selector.raw} resolved to "${nodeType}", attempting fill anyway.` + : undefined; + const { x, y } = centerOfRect(node.rect); + const data = await dispatch( + session.device, + 'fill', + [String(x), String(y), text], + req.flags?.out, + { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + }, + ); + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { + action: 'fill', + }); + const resultPayload: Record = { + ...(data ?? { x, y, text }), + selector: resolved.selector.raw, + selectorChain, + refLabel: resolveRefLabel(node, snapshot.nodes), + }; + if (fillWarning) { + resultPayload.warning = fillWarning; + } + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: resultPayload, + }); + return { ok: true, data: resultPayload }; +} diff --git a/src/daemon/handlers/interaction-flags.ts b/src/daemon/handlers/interaction-flags.ts new file mode 100644 index 000000000..9f6f60eec --- /dev/null +++ b/src/daemon/handlers/interaction-flags.ts @@ -0,0 +1,32 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import type { DaemonResponse } from '../types.ts'; + +const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [ + ['snapshotDepth', '--depth'], + ['snapshotScope', '--scope'], + ['snapshotRaw', '--raw'], +]; + +export function refSnapshotFlagGuardResponse( + command: 'press' | 'fill' | 'get' | 'scrollintoview', + flags: CommandFlags | undefined, +): DaemonResponse | null { + const unsupported = unsupportedRefSnapshotFlags(flags); + if (unsupported.length === 0) return null; + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: `${command} @ref does not support ${unsupported.join(', ')}.`, + }, + }; +} + +export function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): string[] { + if (!flags) return []; + const unsupported: string[] = []; + for (const [key, label] of REF_UNSUPPORTED_FLAG_MAP) { + if (flags[key] !== undefined) unsupported.push(label); + } + return unsupported; +} diff --git a/src/daemon/handlers/interaction-get.ts b/src/daemon/handlers/interaction-get.ts new file mode 100644 index 000000000..d01f1f2d9 --- /dev/null +++ b/src/daemon/handlers/interaction-get.ts @@ -0,0 +1,118 @@ +import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { buildSelectorChainForNode } from '../selectors.ts'; +import { extractNodeText } from '../snapshot-processing.ts'; +import type { DaemonResponse } from '../types.ts'; +import type { InteractionHandlerParams } from './interaction-common.ts'; +import { refSnapshotFlagGuardResponse } from './interaction-flags.ts'; +import { resolveRefTarget } from './interaction-targeting.ts'; +import { resolveSelectorTarget } from './interaction-selector.ts'; + +export async function handleGetCommand(params: InteractionHandlerParams): Promise { + const { req, sessionName, sessionStore, contextFromFlags, dispatch } = params; + const sub = req.positionals?.[0]; + if (sub !== 'text' && sub !== 'attrs') { + return { + ok: false, + error: { code: 'INVALID_ARGS', message: 'get only supports text or attrs' }, + }; + } + const session = sessionStore.get(sessionName); + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + if (!isCommandSupportedOnDevice('get', session.device)) { + return { + ok: false, + error: { code: 'UNSUPPORTED_OPERATION', message: 'get is not supported on this device' }, + }; + } + const refInput = req.positionals?.[1] ?? ''; + if (refInput.startsWith('@')) { + const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('get', req.flags); + if (invalidRefFlagsResponse) return invalidRefFlagsResponse; + const labelCandidate = + req.positionals.length > 2 ? req.positionals.slice(2).join(' ').trim() : ''; + const resolvedRefTarget = resolveRefTarget({ + session, + refInput, + fallbackLabel: labelCandidate, + requireRect: false, + invalidRefMessage: 'get text requires a ref like @e2', + notFoundMessage: `Ref ${refInput} not found`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node } = resolvedRefTarget.target; + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { + action: 'get', + }); + if (sub === 'attrs') { + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { ref, selectorChain }, + }); + return { ok: true, data: { ref, node } }; + } + const text = extractNodeText(node); + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { ref, text, refLabel: text || undefined, selectorChain }, + }); + return { ok: true, data: { ref, text, node } }; + } + + const selectorExpression = req.positionals.slice(1).join(' ').trim(); + if (!selectorExpression) { + return { + ok: false, + error: { code: 'INVALID_ARGS', message: 'get requires @ref or selector expression' }, + }; + } + const resolvedSelectorTarget = await resolveSelectorTarget({ + command: req.command, + selectorExpression, + session, + flags: req.flags, + sessionStore, + contextFromFlags, + interactiveOnly: false, + requireRect: false, + requireUnique: true, + disambiguateAmbiguous: sub === 'text', + dispatch, + }); + if (!resolvedSelectorTarget.ok) return resolvedSelectorTarget.response; + const { resolved } = resolvedSelectorTarget; + const node = resolved.node; + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { + action: 'get', + }); + if (sub === 'attrs') { + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { selector: resolved.selector.raw, selectorChain }, + }); + return { ok: true, data: { selector: resolved.selector.raw, node } }; + } + const text = extractNodeText(node); + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { + text, + refLabel: text || undefined, + selector: resolved.selector.raw, + selectorChain, + }, + }); + return { ok: true, data: { selector: resolved.selector.raw, text, node } }; +} diff --git a/src/daemon/handlers/interaction-is.ts b/src/daemon/handlers/interaction-is.ts new file mode 100644 index 000000000..0b0c4dfa8 --- /dev/null +++ b/src/daemon/handlers/interaction-is.ts @@ -0,0 +1,151 @@ +import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { evaluateIsPredicate, isSupportedPredicate } from '../is-predicates.ts'; +import { + findSelectorChainMatch, + formatSelectorFailure, + parseSelectorChain, + splitIsSelectorArgs, +} from '../selectors.ts'; +import type { DaemonResponse } from '../types.ts'; +import type { InteractionHandlerParams } from './interaction-common.ts'; +import { captureSnapshotForSession } from './interaction-snapshot.ts'; +import { resolveSelectorTarget } from './interaction-selector.ts'; + +export async function handleIsCommand(params: InteractionHandlerParams): Promise { + const { req, sessionName, sessionStore, contextFromFlags, dispatch } = params; + const predicate = (req.positionals?.[0] ?? '').toLowerCase(); + if (!isSupportedPredicate(predicate)) { + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: 'is requires predicate: visible|hidden|exists|editable|selected|text', + }, + }; + } + const session = sessionStore.get(sessionName); + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + if (!isCommandSupportedOnDevice('is', session.device)) { + return { + ok: false, + error: { code: 'UNSUPPORTED_OPERATION', message: 'is is not supported on this device' }, + }; + } + const { split } = splitIsSelectorArgs(req.positionals); + if (!split) { + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: 'is requires a selector expression', + }, + }; + } + const expectedText = split.rest.join(' ').trim(); + if (predicate === 'text' && !expectedText) { + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: 'is text requires expected text value', + }, + }; + } + if (predicate !== 'text' && split.rest.length > 0) { + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: `is ${predicate} does not accept trailing values`, + }, + }; + } + const chain = parseSelectorChain(split.selectorExpression); + const snapshot = await captureSnapshotForSession( + session, + req.flags, + sessionStore, + contextFromFlags, + { interactiveOnly: false }, + dispatch, + ); + if (predicate === 'exists') { + const matched = findSelectorChainMatch(snapshot.nodes, chain, { + platform: session.device.platform, + }); + if (!matched) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: formatSelectorFailure(chain, [], { unique: false }), + }, + }; + } + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { + predicate, + selector: matched.selector.raw, + selectorChain: chain.selectors.map((entry) => entry.raw), + pass: true, + matches: matched.matches, + }, + }); + return { + ok: true, + data: { predicate, pass: true, selector: matched.selector.raw, matches: matched.matches }, + }; + } + + const resolvedSelectorTarget = await resolveSelectorTarget({ + command: 'is', + selectorExpression: split.selectorExpression, + session, + flags: req.flags, + sessionStore, + contextFromFlags, + interactiveOnly: false, + requireRect: false, + requireUnique: true, + disambiguateAmbiguous: false, + dispatch, + }); + if (!resolvedSelectorTarget.ok) return resolvedSelectorTarget.response; + const { resolved } = resolvedSelectorTarget; + const result = evaluateIsPredicate({ + predicate, + node: resolved.node, + expectedText, + platform: session.device.platform, + }); + if (!result.pass) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `is ${predicate} failed for selector ${resolved.selector.raw}: ${result.details}`, + }, + }; + } + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { + predicate, + selector: resolved.selector.raw, + selectorChain: chain.selectors.map((entry) => entry.raw), + pass: true, + text: predicate === 'text' ? result.actualText : undefined, + }, + }); + return { ok: true, data: { predicate, pass: true, selector: resolved.selector.raw } }; +} diff --git a/src/daemon/handlers/interaction-press.ts b/src/daemon/handlers/interaction-press.ts new file mode 100644 index 000000000..bd51a30b2 --- /dev/null +++ b/src/daemon/handlers/interaction-press.ts @@ -0,0 +1,180 @@ +import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { buildSelectorChainForNode } from '../selectors.ts'; +import { findNodeByLabel, resolveRefLabel } from '../snapshot-processing.ts'; +import { findNodeByRef } from '../../utils/snapshot.ts'; +import type { DaemonResponse } from '../types.ts'; +import { refSnapshotFlagGuardResponse } from './interaction-flags.ts'; +import { + parseCoordinateTarget, + resolveRectCenter, + resolveRefTarget, +} from './interaction-targeting.ts'; +import { captureSnapshotForSession } from './interaction-snapshot.ts'; +import { resolveSelectorTarget } from './interaction-selector.ts'; +import type { InteractionHandlerParams } from './interaction-common.ts'; + +export async function handlePressCommand( + params: InteractionHandlerParams, +): Promise { + const { req, sessionName, sessionStore, contextFromFlags, dispatch } = params; + const session = sessionStore.get(sessionName); + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + if (!isCommandSupportedOnDevice('press', session.device)) { + return { + ok: false, + error: { code: 'UNSUPPORTED_OPERATION', message: 'press is not supported on this device' }, + }; + } + const directCoordinates = parseCoordinateTarget(req.positionals ?? []); + if (directCoordinates) { + const data = await dispatch( + session.device, + 'press', + [String(directCoordinates.x), String(directCoordinates.y)], + req.flags?.out, + { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + }, + ); + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [String(directCoordinates.x), String(directCoordinates.y)], + flags: req.flags ?? {}, + result: data ?? { x: directCoordinates.x, y: directCoordinates.y }, + }); + return { ok: true, data: data ?? { x: directCoordinates.x, y: directCoordinates.y } }; + } + + const refInput = req.positionals?.[0] ?? ''; + if (refInput.startsWith('@')) { + const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('press', req.flags); + if (invalidRefFlagsResponse) return invalidRefFlagsResponse; + const fallbackLabel = + req.positionals.length > 1 ? req.positionals.slice(1).join(' ').trim() : ''; + const resolvedRefTarget = resolveRefTarget({ + session, + refInput, + fallbackLabel, + requireRect: true, + invalidRefMessage: 'press requires a ref like @e2', + notFoundMessage: `Ref ${refInput} not found or has no bounds`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref } = resolvedRefTarget.target; + let node = resolvedRefTarget.target.node; + let snapshotNodes = resolvedRefTarget.target.snapshotNodes; + let pressPoint = resolveRectCenter(node.rect); + if (!pressPoint) { + const refreshed = await captureSnapshotForSession( + session, + req.flags, + sessionStore, + contextFromFlags, + { interactiveOnly: true }, + dispatch, + ); + const refNode = findNodeByRef(refreshed.nodes, ref); + const fallbackNode = + fallbackLabel.length > 0 ? findNodeByLabel(refreshed.nodes, fallbackLabel) : null; + const fallbackNodePoint = resolveRectCenter(fallbackNode?.rect); + const refNodePoint = resolveRectCenter(refNode?.rect); + const refreshedNode = refNodePoint + ? refNode + : fallbackNodePoint + ? fallbackNode + : (refNode ?? fallbackNode); + const refreshedPoint = resolveRectCenter(refreshedNode?.rect); + if (refreshedNode && refreshedPoint) { + node = refreshedNode; + snapshotNodes = refreshed.nodes; + pressPoint = refreshedPoint; + } + } + if (!pressPoint) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `Ref ${refInput} not found or has invalid bounds`, + }, + }; + } + const refLabel = resolveRefLabel(node, snapshotNodes); + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { + action: 'click', + }); + const { x, y } = pressPoint; + const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + }); + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { ref, x, y, refLabel, selectorChain }, + }); + return { ok: true, data: { ...(data ?? {}), ref, x, y } }; + } + + const selectorExpression = (req.positionals ?? []).join(' ').trim(); + if (!selectorExpression) { + return { + ok: false, + error: { + code: 'INVALID_ARGS', + message: 'press requires @ref, selector expression, or x y coordinates', + }, + }; + } + const resolvedSelectorTarget = await resolveSelectorTarget({ + command: req.command, + selectorExpression, + session, + flags: req.flags, + sessionStore, + contextFromFlags, + interactiveOnly: true, + requireRect: true, + requireUnique: true, + disambiguateAmbiguous: true, + dispatch, + }); + if (!resolvedSelectorTarget.ok) return resolvedSelectorTarget.response; + const { resolved, snapshot } = resolvedSelectorTarget; + const pressPoint = resolveRectCenter(resolved.node.rect); + if (!pressPoint) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `Selector ${resolved.selector.raw} resolved to invalid bounds`, + }, + }; + } + const { x, y } = pressPoint; + const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + }); + const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, { + action: 'click', + }); + const refLabel = resolveRefLabel(resolved.node, snapshot.nodes); + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { + x, + y, + selector: resolved.selector.raw, + selectorChain, + refLabel, + }, + }); + return { ok: true, data: { ...(data ?? {}), selector: resolved.selector.raw, x, y } }; +} diff --git a/src/daemon/handlers/interaction-scroll.ts b/src/daemon/handlers/interaction-scroll.ts new file mode 100644 index 000000000..8c517460e --- /dev/null +++ b/src/daemon/handlers/interaction-scroll.ts @@ -0,0 +1,112 @@ +import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; +import { buildSelectorChainForNode } from '../selectors.ts'; +import { resolveRefLabel } from '../snapshot-processing.ts'; +import { buildScrollIntoViewPlan, resolveViewportRect } from '../scroll-planner.ts'; +import type { DaemonResponse } from '../types.ts'; +import type { InteractionHandlerParams } from './interaction-common.ts'; +import { refSnapshotFlagGuardResponse } from './interaction-flags.ts'; +import { resolveRefTarget } from './interaction-targeting.ts'; + +export async function handleScrollIntoViewCommand( + params: InteractionHandlerParams, +): Promise { + const { req, sessionName, sessionStore, contextFromFlags, dispatch } = params; + const session = sessionStore.get(sessionName); + if (!session) { + return { + ok: false, + error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, + }; + } + if (!isCommandSupportedOnDevice('scrollintoview', session.device)) { + return { + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: 'scrollintoview is not supported on this device', + }, + }; + } + const targetInput = req.positionals?.[0] ?? ''; + if (!targetInput.startsWith('@')) { + return null; + } + const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('scrollintoview', req.flags); + if (invalidRefFlagsResponse) return invalidRefFlagsResponse; + const fallbackLabel = + req.positionals && req.positionals.length > 1 ? req.positionals.slice(1).join(' ').trim() : ''; + const resolvedRefTarget = resolveRefTarget({ + session, + refInput: targetInput, + fallbackLabel, + requireRect: true, + invalidRefMessage: 'scrollintoview requires a ref like @e2', + notFoundMessage: `Ref ${targetInput} not found or has no bounds`, + }); + if (!resolvedRefTarget.ok) return resolvedRefTarget.response; + const { ref, node, snapshotNodes } = resolvedRefTarget.target; + if (!node.rect) { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: `Ref ${targetInput} not found or has no bounds` }, + }; + } + const viewportRect = resolveViewportRect(snapshotNodes, node.rect); + if (!viewportRect) { + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: `scrollintoview could not infer viewport for ${targetInput}`, + }, + }; + } + const plan = buildScrollIntoViewPlan(node.rect, viewportRect); + const refLabel = resolveRefLabel(node, snapshotNodes); + const selectorChain = buildSelectorChainForNode(node, session.device.platform, { + action: 'get', + }); + if (!plan) { + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { ref, attempts: 0, alreadyVisible: true, refLabel, selectorChain }, + }); + return { ok: true, data: { ref, attempts: 0, alreadyVisible: true } }; + } + const data = await dispatch( + session.device, + 'swipe', + [String(plan.x), String(plan.startY), String(plan.x), String(plan.endY), '16'], + req.flags?.out, + { + ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), + count: plan.count, + pauseMs: 0, + pattern: 'one-way', + }, + ); + sessionStore.recordAction(session, { + command: req.command, + positionals: req.positionals ?? [], + flags: req.flags ?? {}, + result: { + ...(data ?? {}), + ref, + attempts: plan.count, + direction: plan.direction, + refLabel, + selectorChain, + }, + }); + return { + ok: true, + data: { + ...(data ?? {}), + ref, + attempts: plan.count, + direction: plan.direction, + }, + }; +} diff --git a/src/daemon/handlers/interaction-selector.ts b/src/daemon/handlers/interaction-selector.ts new file mode 100644 index 000000000..0c309410f --- /dev/null +++ b/src/daemon/handlers/interaction-selector.ts @@ -0,0 +1,78 @@ +import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; +import { formatSelectorFailure, parseSelectorChain, resolveSelectorChain } from '../selectors.ts'; +import type { DaemonResponse, SessionState } from '../types.ts'; +import type { SessionStore } from '../session-store.ts'; +import { captureSnapshotForSession } from './interaction-snapshot.ts'; +import type { ContextFromFlags } from './interaction-common.ts'; +import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts'; + +export async function resolveSelectorTarget(params: { + command: string; + selectorExpression: string; + session: SessionState; + flags: CommandFlags | undefined; + sessionStore: SessionStore; + contextFromFlags: ContextFromFlags; + interactiveOnly: boolean; + requireRect: boolean; + requireUnique: boolean; + disambiguateAmbiguous: boolean; + dispatch?: typeof dispatchCommand; +}): Promise< + | { + ok: true; + chain: ReturnType; + snapshot: Awaited>; + resolved: NonNullable>>; + } + | { ok: false; response: DaemonResponse } +> { + const { + command, + selectorExpression, + session, + flags, + sessionStore, + contextFromFlags, + interactiveOnly, + requireRect, + requireUnique, + disambiguateAmbiguous, + dispatch, + } = params; + const chain = parseSelectorChain(selectorExpression); + const snapshot = await captureSnapshotForSession( + session, + flags, + sessionStore, + contextFromFlags, + { interactiveOnly }, + dispatch, + ); + const resolved = await withDiagnosticTimer( + 'selector_resolve', + () => + resolveSelectorChain(snapshot.nodes, chain, { + platform: session.device.platform, + requireRect, + requireUnique, + disambiguateAmbiguous, + }), + { command }, + ); + if (!resolved || (requireRect && !resolved.node.rect)) { + return { + ok: false, + response: { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: formatSelectorFailure(chain, resolved?.diagnostics ?? [], { + unique: requireUnique, + }), + }, + }, + }; + } + return { ok: true, chain, snapshot, resolved }; +} diff --git a/src/daemon/handlers/interaction-snapshot.ts b/src/daemon/handlers/interaction-snapshot.ts new file mode 100644 index 000000000..3f05932bf --- /dev/null +++ b/src/daemon/handlers/interaction-snapshot.ts @@ -0,0 +1,41 @@ +import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts'; +import { attachRefs, type RawSnapshotNode } from '../../utils/snapshot.ts'; +import { pruneGroupNodes } from '../snapshot-processing.ts'; +import type { SessionStore } from '../session-store.ts'; +import type { SessionState } from '../types.ts'; +import type { ContextFromFlags } from './interaction-common.ts'; + +export async function captureSnapshotForSession( + session: SessionState, + flags: CommandFlags | undefined, + sessionStore: SessionStore, + contextFromFlags: ContextFromFlags, + options: { interactiveOnly: boolean }, + dispatch: typeof dispatchCommand = dispatchCommand, +) { + const data = (await dispatch(session.device, 'snapshot', [], flags?.out, { + ...contextFromFlags( + { + ...(flags ?? {}), + snapshotInteractiveOnly: options.interactiveOnly, + snapshotCompact: options.interactiveOnly, + }, + session.appBundleId, + session.trace?.outPath, + ), + })) as { + nodes?: RawSnapshotNode[]; + truncated?: boolean; + backend?: 'xctest' | 'android'; + }; + const rawNodes = data?.nodes ?? []; + const nodes = attachRefs(flags?.snapshotRaw ? rawNodes : pruneGroupNodes(rawNodes)); + session.snapshot = { + nodes, + truncated: data?.truncated, + createdAt: Date.now(), + backend: data?.backend, + }; + sessionStore.set(session.name, session); + return session.snapshot; +} diff --git a/src/daemon/handlers/interaction-targeting.ts b/src/daemon/handlers/interaction-targeting.ts new file mode 100644 index 000000000..eca1aba5d --- /dev/null +++ b/src/daemon/handlers/interaction-targeting.ts @@ -0,0 +1,84 @@ +import { + centerOfRect, + findNodeByRef, + normalizeRef, + type Rect, + type SnapshotNode, +} from '../../utils/snapshot.ts'; +import { findNodeByLabel } from '../snapshot-processing.ts'; +import type { DaemonResponse, SessionState } from '../types.ts'; + +export function parseCoordinateTarget(positionals: string[]): { x: number; y: number } | null { + if (positionals.length < 2) return null; + const x = Number(positionals[0]); + const y = Number(positionals[1]); + if (!Number.isFinite(x) || !Number.isFinite(y)) return null; + return { x, y }; +} + +export function resolveRefTarget(params: { + session: SessionState; + refInput: string; + fallbackLabel: string; + requireRect: boolean; + invalidRefMessage: string; + notFoundMessage: string; +}): + | { ok: true; target: { ref: string; node: SnapshotNode; snapshotNodes: SnapshotNode[] } } + | { ok: false; response: DaemonResponse } { + const { session, refInput, fallbackLabel, requireRect, invalidRefMessage, notFoundMessage } = + params; + if (!session.snapshot) { + return { + ok: false, + response: { + ok: false, + error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' }, + }, + }; + } + const ref = normalizeRef(refInput); + if (!ref) { + return { + ok: false, + response: { ok: false, error: { code: 'INVALID_ARGS', message: invalidRefMessage } }, + }; + } + let node = findNodeByRef(session.snapshot.nodes, ref); + if ((!node || (requireRect && !node.rect)) && fallbackLabel.length > 0) { + node = findNodeByLabel(session.snapshot.nodes, fallbackLabel); + } + if (!node || (requireRect && !node.rect)) { + return { + ok: false, + response: { ok: false, error: { code: 'COMMAND_FAILED', message: notFoundMessage } }, + }; + } + return { ok: true, target: { ref, node, snapshotNodes: session.snapshot.nodes } }; +} + +export function resolveRectCenter(rect: Rect | undefined): { x: number; y: number } | null { + const normalized = normalizeRect(rect); + if (!normalized) return null; + const center = centerOfRect(normalized); + if (!Number.isFinite(center.x) || !Number.isFinite(center.y)) return null; + return center; +} + +function normalizeRect(rect: Rect | undefined): Rect | null { + if (!rect) return null; + const x = Number(rect.x); + const y = Number(rect.y); + const width = Number(rect.width); + const height = Number(rect.height); + if ( + !Number.isFinite(x) || + !Number.isFinite(y) || + !Number.isFinite(width) || + !Number.isFinite(height) + ) { + return null; + } + if (width < 0 || height < 0) return null; + return { x, y, width, height }; +} diff --git a/src/daemon/handlers/interaction.ts b/src/daemon/handlers/interaction.ts index 09ad06edf..b5e107b26 100644 --- a/src/daemon/handlers/interaction.ts +++ b/src/daemon/handlers/interaction.ts @@ -1,915 +1,36 @@ -import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts'; -import { isCommandSupportedOnDevice } from '../../core/capabilities.ts'; -import { - attachRefs, - centerOfRect, - findNodeByRef, - normalizeRef, - type RawSnapshotNode, - type Rect, - type SnapshotNode, -} from '../../utils/snapshot.ts'; -import type { DaemonCommandContext } from '../context.ts'; -import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts'; -import { SessionStore } from '../session-store.ts'; -import { evaluateIsPredicate, isSupportedPredicate } from '../is-predicates.ts'; -import { - extractNodeText, - findNodeByLabel, - isFillableType, - pruneGroupNodes, - resolveRefLabel, -} from '../snapshot-processing.ts'; -import { - buildSelectorChainForNode, - findSelectorChainMatch, - formatSelectorFailure, - parseSelectorChain, - resolveSelectorChain, - splitIsSelectorArgs, - splitSelectorFromArgs, -} from '../selectors.ts'; -import { withDiagnosticTimer } from '../../utils/diagnostics.ts'; -import { buildScrollIntoViewPlan, resolveViewportRect } from '../scroll-planner.ts'; - -type ContextFromFlags = ( - flags: CommandFlags | undefined, - appBundleId?: string, - traceLogPath?: string, -) => DaemonCommandContext; - -export async function handleInteractionCommands(params: { - req: DaemonRequest; - sessionName: string; - sessionStore: SessionStore; - contextFromFlags: ContextFromFlags; - dispatch?: typeof dispatchCommand; -}): Promise { - const { req, sessionName, sessionStore, contextFromFlags } = params; - const dispatch = params.dispatch ?? dispatchCommand; - const command = req.command; - - if (command === 'press') { - const session = sessionStore.get(sessionName); - if (!session) { - return { - ok: false, - error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, - }; - } - if (!isCommandSupportedOnDevice('press', session.device)) { - return { - ok: false, - error: { code: 'UNSUPPORTED_OPERATION', message: 'press is not supported on this device' }, - }; - } - const directCoordinates = parseCoordinateTarget(req.positionals ?? []); - if (directCoordinates) { - const data = await dispatch( - session.device, - 'press', - [String(directCoordinates.x), String(directCoordinates.y)], - req.flags?.out, - { - ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - }, - ); - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [String(directCoordinates.x), String(directCoordinates.y)], - flags: req.flags ?? {}, - result: data ?? { x: directCoordinates.x, y: directCoordinates.y }, - }); - return { ok: true, data: data ?? { x: directCoordinates.x, y: directCoordinates.y } }; - } - - const selectorAction = 'click'; - const refInput = req.positionals?.[0] ?? ''; - if (refInput.startsWith('@')) { - const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('press', req.flags); - if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - const fallbackLabel = - req.positionals.length > 1 ? req.positionals.slice(1).join(' ').trim() : ''; - const resolvedRefTarget = resolveRefTarget({ - session, - refInput, - fallbackLabel, - requireRect: true, - invalidRefMessage: `${command} requires a ref like @e2`, - notFoundMessage: `Ref ${refInput} not found or has no bounds`, - }); - if (!resolvedRefTarget.ok) return resolvedRefTarget.response; - const { ref } = resolvedRefTarget.target; - let node = resolvedRefTarget.target.node; - let snapshotNodes = resolvedRefTarget.target.snapshotNodes; - let pressPoint = resolveRectCenter(node.rect); - if (!pressPoint) { - const refreshed = await captureSnapshotForSession( - session, - req.flags, - sessionStore, - contextFromFlags, - { interactiveOnly: true }, - dispatch, - ); - const refNode = findNodeByRef(refreshed.nodes, ref); - const fallbackNode = - fallbackLabel.length > 0 ? findNodeByLabel(refreshed.nodes, fallbackLabel) : null; - const fallbackNodePoint = resolveRectCenter(fallbackNode?.rect); - const refNodePoint = resolveRectCenter(refNode?.rect); - const refreshedNode = refNodePoint - ? refNode - : fallbackNodePoint - ? fallbackNode - : (refNode ?? fallbackNode); - const refreshedPoint = resolveRectCenter(refreshedNode?.rect); - if (refreshedNode && refreshedPoint) { - node = refreshedNode; - snapshotNodes = refreshed.nodes; - pressPoint = refreshedPoint; - } - } - if (!pressPoint) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: `Ref ${refInput} not found or has invalid bounds`, - }, - }; - } - const refLabel = resolveRefLabel(node, snapshotNodes); - const selectorChain = buildSelectorChainForNode(node, session.device.platform, { - action: selectorAction, - }); - const { x, y } = pressPoint; - const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, { - ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - }); - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref, x, y, refLabel, selectorChain }, - }); - return { ok: true, data: { ...(data ?? {}), ref, x, y } }; - } - - const selectorExpression = (req.positionals ?? []).join(' ').trim(); - if (!selectorExpression) { - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: `${command} requires @ref, selector expression, or x y coordinates`, - }, - }; - } - const chain = parseSelectorChain(selectorExpression); - const snapshot = await captureSnapshotForSession( - session, - req.flags, - sessionStore, - contextFromFlags, - { interactiveOnly: true }, - dispatch, - ); - const resolved = await withDiagnosticTimer( - 'selector_resolve', - () => - resolveSelectorChain(snapshot.nodes, chain, { - platform: session.device.platform, - requireRect: true, - requireUnique: true, - disambiguateAmbiguous: true, - }), - { command }, - ); - if (!resolved || !resolved.node.rect) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: formatSelectorFailure(chain, resolved?.diagnostics ?? [], { unique: true }), - }, - }; - } - const pressPoint = resolveRectCenter(resolved.node.rect); - if (!pressPoint) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: `Selector ${resolved.selector.raw} resolved to invalid bounds`, - }, - }; - } - const { x, y } = pressPoint; - const data = await dispatch(session.device, 'press', [String(x), String(y)], req.flags?.out, { - ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - }); - const selectorChain = buildSelectorChainForNode(resolved.node, session.device.platform, { - action: selectorAction, - }); - const refLabel = resolveRefLabel(resolved.node, snapshot.nodes); - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { - x, - y, - selector: resolved.selector.raw, - selectorChain, - refLabel, - }, - }); - return { ok: true, data: { ...(data ?? {}), selector: resolved.selector.raw, x, y } }; - } - - if (command === 'fill') { - const session = sessionStore.get(sessionName); - if (session && !isCommandSupportedOnDevice('fill', session.device)) { - return { - ok: false, - error: { code: 'UNSUPPORTED_OPERATION', message: 'fill is not supported on this device' }, - }; - } - if (req.positionals?.[0]?.startsWith('@')) { - if (!session) { - return { - ok: false, - error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, - }; - } - const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('fill', req.flags); - if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - const labelCandidate = req.positionals.length >= 3 ? req.positionals[1] : ''; - const text = - req.positionals.length >= 3 - ? req.positionals.slice(2).join(' ') - : req.positionals.slice(1).join(' '); - if (!text) { - return { - ok: false, - error: { code: 'INVALID_ARGS', message: 'fill requires text after ref' }, - }; - } - const resolvedRefTarget = resolveRefTarget({ - session, - refInput: req.positionals[0], - fallbackLabel: labelCandidate, - requireRect: true, - invalidRefMessage: 'fill requires a ref like @e2', - notFoundMessage: `Ref ${req.positionals[0]} not found or has no bounds`, - }); - if (!resolvedRefTarget.ok) return resolvedRefTarget.response; - const { ref, node, snapshotNodes } = resolvedRefTarget.target; - if (!node.rect) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: `Ref ${req.positionals[0]} not found or has no bounds`, - }, - }; - } - const nodeType = node.type ?? ''; - const fillWarning = - nodeType && !isFillableType(nodeType, session.device.platform) - ? `fill target ${req.positionals[0]} resolved to "${nodeType}", attempting fill anyway.` - : undefined; - const refLabel = resolveRefLabel(node, snapshotNodes); - const selectorChain = buildSelectorChainForNode(node, session.device.platform, { - action: 'fill', - }); - const { x, y } = centerOfRect(node.rect); - const data = await dispatch( - session.device, - 'fill', - [String(x), String(y), text], - req.flags?.out, - { - ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - }, - ); - const resultPayload: Record = { - ...(data ?? { ref, x, y }), - }; - if (fillWarning) { - resultPayload.warning = fillWarning; - } - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ...resultPayload, refLabel, selectorChain }, - }); - return { ok: true, data: resultPayload }; - } - if (!session) { - return { - ok: false, - error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, - }; - } - const selectorArgs = splitSelectorFromArgs(req.positionals ?? [], { - preferTrailingValue: true, - }); - if (selectorArgs) { - if (selectorArgs.rest.length === 0) { - return { - ok: false, - error: { code: 'INVALID_ARGS', message: 'fill requires text after selector' }, - }; - } - const text = selectorArgs.rest.join(' ').trim(); - if (!text) { - return { - ok: false, - error: { code: 'INVALID_ARGS', message: 'fill requires text after selector' }, - }; - } - const chain = parseSelectorChain(selectorArgs.selectorExpression); - const snapshot = await captureSnapshotForSession( - session, - req.flags, - sessionStore, - contextFromFlags, - { interactiveOnly: true }, - dispatch, - ); - const resolved = await withDiagnosticTimer( - 'selector_resolve', - () => - resolveSelectorChain(snapshot.nodes, chain, { - platform: session.device.platform, - requireRect: true, - requireUnique: true, - disambiguateAmbiguous: true, - }), - { command }, - ); - if (!resolved || !resolved.node.rect) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: formatSelectorFailure(chain, resolved?.diagnostics ?? [], { unique: true }), - }, - }; - } - const node = resolved.node; - const nodeType = node.type ?? ''; - const fillWarning = - nodeType && !isFillableType(nodeType, session.device.platform) - ? `fill target ${resolved.selector.raw} resolved to "${nodeType}", attempting fill anyway.` - : undefined; - const { x, y } = centerOfRect(resolved.node.rect); - const data = await dispatch( - session.device, - 'fill', - [String(x), String(y), text], - req.flags?.out, - { - ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - }, - ); - const selectorChain = buildSelectorChainForNode(node, session.device.platform, { - action: 'fill', - }); - const resultPayload: Record = { - ...(data ?? { x, y, text }), - selector: resolved.selector.raw, - selectorChain, - refLabel: resolveRefLabel(node, snapshot.nodes), - }; - if (fillWarning) { - resultPayload.warning = fillWarning; - } - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: resultPayload, - }); - return { ok: true, data: resultPayload }; - } - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: 'fill requires x y text, @ref text, or selector text', - }, - }; - } - - if (command === 'get') { - const sub = req.positionals?.[0]; - if (sub !== 'text' && sub !== 'attrs') { - return { - ok: false, - error: { code: 'INVALID_ARGS', message: 'get only supports text or attrs' }, - }; - } - const session = sessionStore.get(sessionName); - if (!session) { - return { - ok: false, - error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, - }; - } - if (!isCommandSupportedOnDevice('get', session.device)) { - return { - ok: false, - error: { code: 'UNSUPPORTED_OPERATION', message: 'get is not supported on this device' }, - }; - } - const refInput = req.positionals?.[1] ?? ''; - if (refInput.startsWith('@')) { - const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('get', req.flags); - if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - const labelCandidate = - req.positionals.length > 2 ? req.positionals.slice(2).join(' ').trim() : ''; - const resolvedRefTarget = resolveRefTarget({ - session, - refInput, - fallbackLabel: labelCandidate, - requireRect: false, - invalidRefMessage: 'get text requires a ref like @e2', - notFoundMessage: `Ref ${refInput} not found`, - }); - if (!resolvedRefTarget.ok) return resolvedRefTarget.response; - const { ref, node } = resolvedRefTarget.target; - const selectorChain = buildSelectorChainForNode(node, session.device.platform, { - action: 'get', - }); - if (sub === 'attrs') { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref, selectorChain }, - }); - return { ok: true, data: { ref, node } }; - } - const text = extractNodeText(node); - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref, text, refLabel: text || undefined, selectorChain }, - }); - return { ok: true, data: { ref, text, node } }; - } - - const selectorExpression = req.positionals.slice(1).join(' ').trim(); - if (!selectorExpression) { - return { - ok: false, - error: { code: 'INVALID_ARGS', message: 'get requires @ref or selector expression' }, - }; - } - const chain = parseSelectorChain(selectorExpression); - const snapshot = await captureSnapshotForSession( - session, - req.flags, - sessionStore, - contextFromFlags, - { interactiveOnly: false }, - dispatch, - ); - const resolved = await withDiagnosticTimer( - 'selector_resolve', - () => - resolveSelectorChain(snapshot.nodes, chain, { - platform: session.device.platform, - requireRect: false, - requireUnique: true, - disambiguateAmbiguous: sub === 'text', - }), - { command }, - ); - if (!resolved) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: formatSelectorFailure(chain, [], { unique: true }), - }, - }; - } - const node = resolved.node; - const selectorChain = buildSelectorChainForNode(node, session.device.platform, { - action: 'get', - }); - if (sub === 'attrs') { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { selector: resolved.selector.raw, selectorChain }, - }); - return { ok: true, data: { selector: resolved.selector.raw, node } }; - } - const text = extractNodeText(node); - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { - text, - refLabel: text || undefined, - selector: resolved.selector.raw, - selectorChain, - }, - }); - return { ok: true, data: { selector: resolved.selector.raw, text, node } }; - } - - if (command === 'is') { - const predicate = (req.positionals?.[0] ?? '').toLowerCase(); - if (!isSupportedPredicate(predicate)) { - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: 'is requires predicate: visible|hidden|exists|editable|selected|text', - }, - }; - } - const session = sessionStore.get(sessionName); - if (!session) { - return { - ok: false, - error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, - }; - } - if (!isCommandSupportedOnDevice('is', session.device)) { - return { - ok: false, - error: { code: 'UNSUPPORTED_OPERATION', message: 'is is not supported on this device' }, - }; - } - const { split } = splitIsSelectorArgs(req.positionals); - if (!split) { - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: 'is requires a selector expression', - }, - }; - } - const expectedText = split.rest.join(' ').trim(); - if (predicate === 'text' && !expectedText) { - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: 'is text requires expected text value', - }, - }; - } - if (predicate !== 'text' && split.rest.length > 0) { - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: `is ${predicate} does not accept trailing values`, - }, - }; - } - const chain = parseSelectorChain(split.selectorExpression); - const snapshot = await captureSnapshotForSession( - session, - req.flags, - sessionStore, - contextFromFlags, - { interactiveOnly: false }, - dispatch, - ); - if (predicate === 'exists') { - const matched = findSelectorChainMatch(snapshot.nodes, chain, { - platform: session.device.platform, - }); - if (!matched) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: formatSelectorFailure(chain, [], { unique: false }), - }, - }; - } - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { - predicate, - selector: matched.selector.raw, - selectorChain: chain.selectors.map((entry) => entry.raw), - pass: true, - matches: matched.matches, - }, - }); - return { - ok: true, - data: { predicate, pass: true, selector: matched.selector.raw, matches: matched.matches }, - }; - } - - const resolved = await withDiagnosticTimer( - 'selector_resolve', - () => - resolveSelectorChain(snapshot.nodes, chain, { - platform: session.device.platform, - requireUnique: true, - }), - { command: 'is', predicate }, - ); - if (!resolved) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: formatSelectorFailure(chain, [], { unique: true }), - }, - }; - } - const result = evaluateIsPredicate({ - predicate, - node: resolved.node, - expectedText, - platform: session.device.platform, - }); - if (!result.pass) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: `is ${predicate} failed for selector ${resolved.selector.raw}: ${result.details}`, - }, - }; - } - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { - predicate, - selector: resolved.selector.raw, - selectorChain: chain.selectors.map((entry) => entry.raw), - pass: true, - text: predicate === 'text' ? result.actualText : undefined, - }, - }); - return { ok: true, data: { predicate, pass: true, selector: resolved.selector.raw } }; - } - - if (command === 'scrollintoview') { - const session = sessionStore.get(sessionName); - if (!session) { - return { - ok: false, - error: { code: 'SESSION_NOT_FOUND', message: 'No active session. Run open first.' }, - }; - } - if (!isCommandSupportedOnDevice('scrollintoview', session.device)) { - return { - ok: false, - error: { - code: 'UNSUPPORTED_OPERATION', - message: 'scrollintoview is not supported on this device', - }, - }; - } - const targetInput = req.positionals?.[0] ?? ''; - if (!targetInput.startsWith('@')) { - return null; - } - const invalidRefFlagsResponse = refSnapshotFlagGuardResponse('scrollintoview', req.flags); - if (invalidRefFlagsResponse) return invalidRefFlagsResponse; - const fallbackLabel = - req.positionals && req.positionals.length > 1 - ? req.positionals.slice(1).join(' ').trim() - : ''; - const resolvedRefTarget = resolveRefTarget({ - session, - refInput: targetInput, - fallbackLabel, - requireRect: true, - invalidRefMessage: 'scrollintoview requires a ref like @e2', - notFoundMessage: `Ref ${targetInput} not found or has no bounds`, - }); - if (!resolvedRefTarget.ok) return resolvedRefTarget.response; - const { ref, node, snapshotNodes } = resolvedRefTarget.target; - if (!node.rect) { - return { - ok: false, - error: { code: 'COMMAND_FAILED', message: `Ref ${targetInput} not found or has no bounds` }, - }; - } - const viewportRect = resolveViewportRect(snapshotNodes, node.rect); - if (!viewportRect) { - return { - ok: false, - error: { - code: 'COMMAND_FAILED', - message: `scrollintoview could not infer viewport for ${targetInput}`, - }, - }; - } - const plan = buildScrollIntoViewPlan(node.rect, viewportRect); - const refLabel = resolveRefLabel(node, snapshotNodes); - const selectorChain = buildSelectorChainForNode(node, session.device.platform, { - action: 'get', - }); - if (!plan) { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref, attempts: 0, alreadyVisible: true, refLabel, selectorChain }, - }); - return { ok: true, data: { ref, attempts: 0, alreadyVisible: true } }; - } - const data = await dispatch( - session.device, - 'swipe', - [String(plan.x), String(plan.startY), String(plan.x), String(plan.endY), '16'], - req.flags?.out, - { - ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath), - count: plan.count, - pauseMs: 0, - pattern: 'one-way', - }, - ); - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { - ...(data ?? {}), - ref, - attempts: plan.count, - direction: plan.direction, - refLabel, - selectorChain, - }, - }); - return { - ok: true, - data: { - ...(data ?? {}), - ref, - attempts: plan.count, - direction: plan.direction, - }, - }; - } - - return null; -} - -async function captureSnapshotForSession( - session: SessionState, - flags: CommandFlags | undefined, - sessionStore: SessionStore, - contextFromFlags: ContextFromFlags, - options: { interactiveOnly: boolean }, - dispatch: typeof dispatchCommand = dispatchCommand, -) { - const data = (await dispatch(session.device, 'snapshot', [], flags?.out, { - ...contextFromFlags( - { - ...(flags ?? {}), - snapshotInteractiveOnly: options.interactiveOnly, - snapshotCompact: options.interactiveOnly, - }, - session.appBundleId, - session.trace?.outPath, - ), - })) as { - nodes?: RawSnapshotNode[]; - truncated?: boolean; - backend?: 'xctest' | 'android'; +import { dispatchCommand } from '../../core/dispatch.ts'; +import type { DaemonResponse } from '../types.ts'; +import { handleFillCommand } from './interaction-fill.ts'; +import { handleGetCommand } from './interaction-get.ts'; +import type { InteractionHandlerParams } from './interaction-common.ts'; +import { handleIsCommand } from './interaction-is.ts'; +import { handlePressCommand } from './interaction-press.ts'; +import { handleScrollIntoViewCommand } from './interaction-scroll.ts'; + +export { unsupportedRefSnapshotFlags } from './interaction-flags.ts'; + +export async function handleInteractionCommands( + params: Omit & { + dispatch?: typeof dispatchCommand; + }, +): Promise { + const handlerParams: InteractionHandlerParams = { + ...params, + dispatch: params.dispatch ?? dispatchCommand, }; - const rawNodes = data?.nodes ?? []; - const nodes = attachRefs(flags?.snapshotRaw ? rawNodes : pruneGroupNodes(rawNodes)); - session.snapshot = { - nodes, - truncated: data?.truncated, - createdAt: Date.now(), - backend: data?.backend, - }; - sessionStore.set(session.name, session); - return session.snapshot; -} - -const REF_UNSUPPORTED_FLAG_MAP: ReadonlyArray<[keyof CommandFlags, string]> = [ - ['snapshotDepth', '--depth'], - ['snapshotScope', '--scope'], - ['snapshotRaw', '--raw'], -]; - -function refSnapshotFlagGuardResponse( - command: 'press' | 'fill' | 'get' | 'scrollintoview', - flags: CommandFlags | undefined, -): DaemonResponse | null { - const unsupported = unsupportedRefSnapshotFlags(flags); - if (unsupported.length === 0) return null; - return { - ok: false, - error: { - code: 'INVALID_ARGS', - message: `${command} @ref does not support ${unsupported.join(', ')}.`, - }, - }; -} - -function parseCoordinateTarget(positionals: string[]): { x: number; y: number } | null { - if (positionals.length < 2) return null; - const x = Number(positionals[0]); - const y = Number(positionals[1]); - if (!Number.isFinite(x) || !Number.isFinite(y)) return null; - return { x, y }; -} - -export function unsupportedRefSnapshotFlags(flags: CommandFlags | undefined): string[] { - if (!flags) return []; - const unsupported: string[] = []; - for (const [key, label] of REF_UNSUPPORTED_FLAG_MAP) { - if (flags[key] !== undefined) unsupported.push(label); - } - return unsupported; -} - -function resolveRefTarget(params: { - session: SessionState; - refInput: string; - fallbackLabel: string; - requireRect: boolean; - invalidRefMessage: string; - notFoundMessage: string; -}): - | { ok: true; target: { ref: string; node: SnapshotNode; snapshotNodes: SnapshotNode[] } } - | { ok: false; response: DaemonResponse } { - const { session, refInput, fallbackLabel, requireRect, invalidRefMessage, notFoundMessage } = - params; - if (!session.snapshot) { - return { - ok: false, - response: { - ok: false, - error: { code: 'INVALID_ARGS', message: 'No snapshot in session. Run snapshot first.' }, - }, - }; - } - const ref = normalizeRef(refInput); - if (!ref) { - return { - ok: false, - response: { ok: false, error: { code: 'INVALID_ARGS', message: invalidRefMessage } }, - }; - } - let node = findNodeByRef(session.snapshot.nodes, ref); - if ((!node || (requireRect && !node.rect)) && fallbackLabel.length > 0) { - node = findNodeByLabel(session.snapshot.nodes, fallbackLabel); - } - if (!node || (requireRect && !node.rect)) { - return { - ok: false, - response: { ok: false, error: { code: 'COMMAND_FAILED', message: notFoundMessage } }, - }; - } - return { ok: true, target: { ref, node, snapshotNodes: session.snapshot.nodes } }; -} -function resolveRectCenter(rect: Rect | undefined): { x: number; y: number } | null { - const normalized = normalizeRect(rect); - if (!normalized) return null; - const center = centerOfRect(normalized); - if (!Number.isFinite(center.x) || !Number.isFinite(center.y)) return null; - return center; -} - -function normalizeRect(rect: Rect | undefined): Rect | null { - if (!rect) return null; - const x = Number(rect.x); - const y = Number(rect.y); - const width = Number(rect.width); - const height = Number(rect.height); - if ( - !Number.isFinite(x) || - !Number.isFinite(y) || - !Number.isFinite(width) || - !Number.isFinite(height) - ) { - return null; + switch (params.req.command) { + case 'press': + return await handlePressCommand(handlerParams); + case 'fill': + return await handleFillCommand(handlerParams); + case 'get': + return await handleGetCommand(handlerParams); + case 'is': + return await handleIsCommand(handlerParams); + case 'scrollintoview': + return await handleScrollIntoViewCommand(handlerParams); + default: + return null; } - if (width < 0 || height < 0) return null; - return { x, y, width, height }; } From c8771a8c8a744f3c38b1bcda998f541fc712d231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 22 Mar 2026 19:40:44 +0100 Subject: [PATCH 2/2] fix: avoid duplicate snapshot in is handler --- .../handlers/__tests__/interaction.test.ts | 50 +++++++++++++++++++ src/daemon/handlers/interaction-is.ts | 16 +++--- src/daemon/handlers/interaction-selector.ts | 2 +- src/daemon/handlers/interaction-snapshot.ts | 3 +- 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 24b9c2f98..39fe0834d 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -524,3 +524,53 @@ test('scrollintoview @ref does not run post-scroll verification snapshot', async assert.equal(response.ok, true); assert.equal(snapshotCallCount, 0); }); + +test('is visible captures one snapshot before evaluating selector predicate', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'default'; + sessionStore.set(sessionName, makeSession(sessionName)); + + let snapshotCallCount = 0; + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'is', + positionals: ['visible', 'id=auth_continue'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + dispatch: async (_device, command) => { + if (command === 'snapshot') { + snapshotCallCount += 1; + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + identifier: 'auth_continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + visible: true, + }, + ], + backend: 'xctest', + }; + } + throw new Error(`unexpected command: ${command}`); + }, + }); + + assert.ok(response); + assert.equal(response.ok, true); + assert.equal(snapshotCallCount, 1); + if (response.ok) { + assert.equal(response.data?.predicate, 'visible'); + assert.equal(response.data?.pass, true); + assert.equal(response.data?.selector, 'id=auth_continue'); + } +}); diff --git a/src/daemon/handlers/interaction-is.ts b/src/daemon/handlers/interaction-is.ts index 0b0c4dfa8..1f083019c 100644 --- a/src/daemon/handlers/interaction-is.ts +++ b/src/daemon/handlers/interaction-is.ts @@ -66,15 +66,15 @@ export async function handleIsCommand(params: InteractionHandlerParams): Promise }; } const chain = parseSelectorChain(split.selectorExpression); - const snapshot = await captureSnapshotForSession( - session, - req.flags, - sessionStore, - contextFromFlags, - { interactiveOnly: false }, - dispatch, - ); if (predicate === 'exists') { + const snapshot = await captureSnapshotForSession( + session, + req.flags, + sessionStore, + contextFromFlags, + { interactiveOnly: false }, + dispatch, + ); const matched = findSelectorChainMatch(snapshot.nodes, chain, { platform: session.device.platform, }); diff --git a/src/daemon/handlers/interaction-selector.ts b/src/daemon/handlers/interaction-selector.ts index 0c309410f..ec9bf75f5 100644 --- a/src/daemon/handlers/interaction-selector.ts +++ b/src/daemon/handlers/interaction-selector.ts @@ -17,7 +17,7 @@ export async function resolveSelectorTarget(params: { requireRect: boolean; requireUnique: boolean; disambiguateAmbiguous: boolean; - dispatch?: typeof dispatchCommand; + dispatch: typeof dispatchCommand; }): Promise< | { ok: true; diff --git a/src/daemon/handlers/interaction-snapshot.ts b/src/daemon/handlers/interaction-snapshot.ts index 3f05932bf..093aae379 100644 --- a/src/daemon/handlers/interaction-snapshot.ts +++ b/src/daemon/handlers/interaction-snapshot.ts @@ -3,6 +3,7 @@ import { attachRefs, type RawSnapshotNode } from '../../utils/snapshot.ts'; import { pruneGroupNodes } from '../snapshot-processing.ts'; import type { SessionStore } from '../session-store.ts'; import type { SessionState } from '../types.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; import type { ContextFromFlags } from './interaction-common.ts'; export async function captureSnapshotForSession( @@ -12,7 +13,7 @@ export async function captureSnapshotForSession( contextFromFlags: ContextFromFlags, options: { interactiveOnly: boolean }, dispatch: typeof dispatchCommand = dispatchCommand, -) { +): Promise { const data = (await dispatch(session.device, 'snapshot', [], flags?.out, { ...contextFromFlags( {