From 7833e63fc9a7192246d44c5a49c89b999ddfb91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 30 May 2026 13:24:54 +0200 Subject: [PATCH 1/2] refactor: converge Maestro gesture handling --- .../__tests__/runtime-geometry.test.ts | 25 ++++- .../__tests__/runtime-interactions.test.ts | 81 +++++++++++++- src/compat/maestro/runtime-geometry.ts | 25 +++-- src/compat/maestro/runtime-interactions.ts | 103 ++++++++---------- src/core/__tests__/scroll-gesture.test.ts | 34 +++++- src/core/scroll-gesture.ts | 70 ++++++++++++ 6 files changed, 270 insertions(+), 68 deletions(-) diff --git a/src/compat/maestro/__tests__/runtime-geometry.test.ts b/src/compat/maestro/__tests__/runtime-geometry.test.ts index 3be3cce8c..32a53375b 100644 --- a/src/compat/maestro/__tests__/runtime-geometry.test.ts +++ b/src/compat/maestro/__tests__/runtime-geometry.test.ts @@ -1,5 +1,5 @@ import { expect, test } from 'vitest'; -import { pointForMaestroTapOnTarget } from '../runtime-geometry.ts'; +import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from '../runtime-geometry.ts'; test('pointForMaestroTapOnTarget biases large scroll-area text containers toward the visible label', () => { const point = pointForMaestroTapOnTarget( @@ -38,3 +38,26 @@ test('pointForMaestroTapOnTarget centers tall Android bottom-tab containers', () expect(point).toEqual({ x: 675, y: 2164 }); }); + +test('swipeCoordinatesFromTarget preserves Maestro target-relative swipe distance', () => { + const swipe = swipeCoordinatesFromTarget( + { + node: { + index: 12, + ref: 'e12', + type: 'Cell', + label: 'Card', + rect: { x: 100, y: 200, width: 100, height: 80 }, + }, + rect: { x: 100, y: 200, width: 100, height: 80 }, + frame: { referenceWidth: 402, referenceHeight: 874 }, + }, + 'right', + ); + + expect(swipe).toEqual({ + ok: true, + start: { x: 150, y: 240 }, + end: { x: 300, y: 240 }, + }); +}); diff --git a/src/compat/maestro/__tests__/runtime-interactions.test.ts b/src/compat/maestro/__tests__/runtime-interactions.test.ts index c0d2f43f4..e18754fb3 100644 --- a/src/compat/maestro/__tests__/runtime-interactions.test.ts +++ b/src/compat/maestro/__tests__/runtime-interactions.test.ts @@ -1,7 +1,11 @@ import { expect, test } from 'vitest'; import type { DaemonRequest, DaemonResponse } from '../../../daemon/types.ts'; import type { SnapshotState } from '../../../utils/snapshot.ts'; -import { invokeMaestroSwipeScreen, invokeMaestroTapOn } from '../runtime-interactions.ts'; +import { + invokeMaestroSwipeScreen, + invokeMaestroTapOn, + invokeMaestroTapPointPercent, +} from '../runtime-interactions.ts'; test('invokeMaestroTapOn resolves mutating taps from the current raw snapshot', async () => { const selector = @@ -59,6 +63,81 @@ test('invokeMaestroSwipeScreen uses an Android content-lane directional swipe', expect(swipes).toEqual([['864', '1521', '216', '1521', '300']]); }); +test('invokeMaestroSwipeScreen preserves vertical percentage endpoints', async () => { + const swipes: string[][] = []; + const response = await invokeMaestroSwipeScreen({ + baseReq: { + token: 'test', + session: 'article', + flags: { platform: 'ios' }, + }, + positionals: ['percent', '50', '75', '50', '35', '300'], + invoke: async (req: DaemonRequest): Promise => { + if (req.command === 'snapshot') { + return { ok: true, data: fullScreenSnapshot(400, 800) }; + } + if (req.command === 'swipe') { + swipes.push(req.positionals ?? []); + return { ok: true, data: {} }; + } + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + expect(response.ok).toBe(true); + expect(swipes).toEqual([['200', '600', '200', '280', '300']]); +}); + +test('invokeMaestroSwipeScreen keeps Android horizontal percentage swipes on the content lane', async () => { + const swipes: string[][] = []; + const response = await invokeMaestroSwipeScreen({ + baseReq: { + token: 'test', + session: 'pager', + flags: { platform: 'android' }, + }, + positionals: ['percent', '90', '50', '10', '50', '300'], + invoke: async (req: DaemonRequest): Promise => { + if (req.command === 'snapshot') { + return { ok: true, data: fullScreenSnapshot(390, 600) }; + } + if (req.command === 'swipe') { + swipes.push(req.positionals ?? []); + return { ok: true, data: {} }; + } + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + expect(response.ok).toBe(true); + expect(swipes).toEqual([['351', '390', '39', '390', '300']]); +}); + +test('invokeMaestroTapPointPercent shares percentage point geometry without clamping', async () => { + const clicks: string[][] = []; + const response = await invokeMaestroTapPointPercent({ + baseReq: { + token: 'test', + session: 'article', + flags: { platform: 'ios' }, + }, + positionals: ['125', '-10'], + invoke: async (req: DaemonRequest): Promise => { + if (req.command === 'snapshot') { + return { ok: true, data: fullScreenSnapshot(400, 800) }; + } + if (req.command === 'click') { + clicks.push(req.positionals ?? []); + return { ok: true, data: {} }; + } + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + expect(response.ok).toBe(true); + expect(clicks).toEqual([['500', '-80']]); +}); + function currentBreadcrumbSnapshot(): SnapshotState { return { createdAt: Date.now(), diff --git a/src/compat/maestro/runtime-geometry.ts b/src/compat/maestro/runtime-geometry.ts index 0a3027ea3..83d2a8bbf 100644 --- a/src/compat/maestro/runtime-geometry.ts +++ b/src/compat/maestro/runtime-geometry.ts @@ -1,3 +1,4 @@ +import { clampGestureCoordinate } from '../../core/scroll-gesture.ts'; import type { Rect, SnapshotNode } from '../../utils/snapshot.ts'; import { interiorCoordinate, pointInsideRect } from '../../utils/rect-center.ts'; import { normalizeType } from '../../utils/snapshot-processing.ts'; @@ -39,25 +40,37 @@ export function swipeCoordinatesFromTarget( return { ok: true, start: center, - end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, + end: { + x: center.x, + y: clampGestureCoordinate(center.y - verticalDistance, minY, maxY + minY), + }, }; case 'down': return { ok: true, start: center, - end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, + end: { + x: center.x, + y: clampGestureCoordinate(center.y + verticalDistance, minY, maxY + minY), + }, }; case 'left': return { ok: true, start: center, - end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, + end: { + x: clampGestureCoordinate(center.x - horizontalDistance, minX, maxX + minX), + y: center.y, + }, }; case 'right': return { ok: true, start: center, - end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, + end: { + x: clampGestureCoordinate(center.x + horizontalDistance, minX, maxX + minX), + y: center.y, + }, }; default: return { ok: false, message: 'swipe.label direction must be up, down, left, or right.' }; @@ -94,10 +107,6 @@ function swipeDistance(frameSize: number | undefined, rectSize: number): number ); } -function clampCoordinate(value: number, min: number, max: number): number { - return Math.round(Math.min(max, Math.max(min, value))); -} - function shouldBiasMaestroVisibleTextTap( node: SnapshotNode, isVisibleTextSelector: boolean, diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index 1a9db0142..fbe0ee5a6 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -1,5 +1,11 @@ import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; import type { DaemonRequest, DaemonResponse } from '../../daemon/types.ts'; +import { + buildSwipeGesturePlan, + clampGesturePoint, + pointFromPercent, + type ScrollDirection, +} from '../../core/scroll-gesture.ts'; import type { ReplayVarScope } from '../../replay/vars.ts'; import { sleep } from '../../utils/timeouts.ts'; import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; @@ -122,13 +128,11 @@ export async function invokeMaestroTapPointPercent(params: { ); } + const point = pointFromPercent(frame, xPercent, yPercent); return await params.invoke({ ...params.baseReq, command: 'click', - positionals: [ - String(Math.round((frame.referenceWidth * xPercent) / 100)), - String(Math.round((frame.referenceHeight * yPercent) / 100)), - ], + positionals: [String(point.x), String(point.y)], }); } @@ -263,22 +267,12 @@ function resolveDirectionalScreenSwipe( response: errorResponse('INVALID_ARGS', 'Maestro direction swipe requires a direction.'), }; } - const point = (xPercent: number, yPercent: number) => percentPoint(frame, xPercent, yPercent, 8); switch (direction) { case 'up': - return { ok: true, start: point(50, 80), end: point(50, 20), durationMs }; case 'down': - return { ok: true, start: point(50, 20), end: point(50, 80), durationMs }; - case 'left': { - const [startX, endX] = androidHorizontalDirectionalSwipeX(platform, 80, 20); - const yPercent = androidHorizontalContentSwipeY(platform, startX, 50, endX, 50); - return { ok: true, start: point(startX, yPercent), end: point(endX, yPercent), durationMs }; - } - case 'right': { - const [startX, endX] = androidHorizontalDirectionalSwipeX(platform, 20, 80); - const yPercent = androidHorizontalContentSwipeY(platform, startX, 50, endX, 50); - return { ok: true, start: point(startX, yPercent), end: point(endX, yPercent), durationMs }; - } + case 'left': + case 'right': + return buildMaestroDirectionalScreenSwipe(direction, frame, platform, durationMs); default: return { ok: false, @@ -290,13 +284,33 @@ function resolveDirectionalScreenSwipe( } } -function androidHorizontalDirectionalSwipeX( +function buildMaestroDirectionalScreenSwipe( + direction: ScrollDirection, + frame: { referenceWidth: number; referenceHeight: number }, platform: string, - startX: number, - endX: number, -): [number, number] { - if (platform !== 'android') return [startX, endX]; - return startX < endX ? [20, 80] : [80, 20]; + durationMs: string | undefined, +): MaestroScreenSwipeResolution { + const plan = buildSwipeGesturePlan({ + direction, + amount: 0.6, + referenceWidth: frame.referenceWidth, + referenceHeight: frame.referenceHeight, + }); + const start = clampGesturePoint({ x: plan.x1, y: plan.y1 }, frame, 8); + const end = clampGesturePoint({ x: plan.x2, y: plan.y2 }, frame, 8); + + if ((direction === 'left' || direction === 'right') && platform === 'android') { + const contentLaneY = pointFromPercent(frame, 50, 65, { marginPx: 8 }).y; + start.y = contentLaneY; + end.y = contentLaneY; + } + + return { + ok: true, + start, + end, + durationMs, + }; } function resolvePercentScreenSwipe( @@ -313,55 +327,30 @@ function resolvePercentScreenSwipe( }; } const [x1, y1, x2, y2] = values as [number, number, number, number]; - const adjustedY = androidHorizontalContentSwipeY(platform, x1, y1, x2, y2); + const lane = maestroHorizontalContentSwipeLanePercent(platform, x1, y1, x2, y2); return { ok: true, - start: percentPoint(frame, x1, adjustedY, 1), - end: percentPoint(frame, x2, adjustedY, 1), + start: pointFromPercent(frame, x1, lane.startY, { marginPx: 1 }), + end: pointFromPercent(frame, x2, lane.endY, { marginPx: 1 }), durationMs, }; } -function androidHorizontalContentSwipeY( +function maestroHorizontalContentSwipeLanePercent( platform: string, x1: number, y1: number, x2: number, y2: number, -): number { - if (platform !== 'android') return y2; - if (y1 !== y2 || y1 !== 50) return y2; - if (Math.abs(x2 - x1) < 30) return y2; +): { startY: number; endY: number } { + if (platform !== 'android') return { startY: y1, endY: y2 }; + if (y1 !== y2 || y1 !== 50) return { startY: y1, endY: y2 }; + if (Math.abs(x2 - x1) < 30) return { startY: y1, endY: y2 }; // Maestro's Android driver treats 50% horizontal swipes as content swipes. // Raw `adb input swipe` at the physical screen midpoint can land above // horizontally paged content in React Native layouts, so use a lower content // lane for full-width horizontal Maestro percentage swipes. - return 65; -} - -function percentPoint( - frame: { referenceWidth: number; referenceHeight: number }, - xPercent: number, - yPercent: number, - marginPx: number, -): { x: number; y: number } { - return { - x: clampPoint( - Math.round((frame.referenceWidth * xPercent) / 100), - marginPx, - frame.referenceWidth, - ), - y: clampPoint( - Math.round((frame.referenceHeight * yPercent) / 100), - marginPx, - frame.referenceHeight, - ), - }; -} - -function clampPoint(value: number, marginPx: number, size: number): number { - const max = Math.max(marginPx, size - marginPx); - return Math.min(max, Math.max(marginPx, value)); + return { startY: 65, endY: 65 }; } async function probeMaestroScrollVisibility( diff --git a/src/core/__tests__/scroll-gesture.test.ts b/src/core/__tests__/scroll-gesture.test.ts index c196703aa..70ac2075b 100644 --- a/src/core/__tests__/scroll-gesture.test.ts +++ b/src/core/__tests__/scroll-gesture.test.ts @@ -1,7 +1,11 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; import { AppError } from '../../utils/errors.ts'; -import { buildScrollGesturePlan } from '../scroll-gesture.ts'; +import { + buildScrollGesturePlan, + buildSwipeGesturePlan, + pointFromPercent, +} from '../scroll-gesture.ts'; test('buildScrollGesturePlan maps relative amount to viewport travel', () => { const plan = buildScrollGesturePlan({ @@ -54,3 +58,31 @@ test('buildScrollGesturePlan rejects invalid amounts', () => { /amount must be a positive number/i.test(error.message), ); }); + +test('buildSwipeGesturePlan maps finger direction through the shared scroll planner', () => { + const plan = buildSwipeGesturePlan({ + direction: 'left', + amount: 0.6, + referenceWidth: 400, + referenceHeight: 800, + }); + + assert.deepEqual(plan, { + direction: 'left', + x1: 320, + y1: 400, + x2: 80, + y2: 400, + referenceWidth: 400, + referenceHeight: 800, + amount: 0.6, + pixels: 240, + }); +}); + +test('pointFromPercent preserves unclamped percentages and clamps when a margin is requested', () => { + const frame = { referenceWidth: 400, referenceHeight: 800 }; + + assert.deepEqual(pointFromPercent(frame, 125, -10), { x: 500, y: -80 }); + assert.deepEqual(pointFromPercent(frame, 100, 0, { marginPx: 8 }), { x: 392, y: 8 }); +}); diff --git a/src/core/scroll-gesture.ts b/src/core/scroll-gesture.ts index 8e19fc7b2..6e35f2003 100644 --- a/src/core/scroll-gesture.ts +++ b/src/core/scroll-gesture.ts @@ -2,6 +2,16 @@ import { AppError } from '../utils/errors.ts'; export type ScrollDirection = 'up' | 'down' | 'left' | 'right'; +export type GestureReferenceFrame = { + referenceWidth: number; + referenceHeight: number; +}; + +export type GesturePoint = { + x: number; + y: number; +}; + export type ScrollGestureOptions = { direction: ScrollDirection; amount?: number; @@ -22,6 +32,12 @@ export type ScrollGesturePlan = { pixels: number; }; +export type SwipeGestureOptions = ScrollGestureOptions; + +export type SwipeGesturePlan = Omit & { + direction: ScrollDirection; +}; + const DEFAULT_SCROLL_AMOUNT = 0.6; const DEFAULT_EDGE_PADDING_FRACTION = 0.05; @@ -64,6 +80,42 @@ export function buildScrollGesturePlan(options: ScrollGestureOptions): ScrollGes } } +export function buildSwipeGesturePlan(options: SwipeGestureOptions): SwipeGesturePlan { + const scrollPlan = buildScrollGesturePlan({ + ...options, + direction: scrollDirectionForFingerSwipe(options.direction), + }); + return { + ...scrollPlan, + direction: options.direction, + }; +} + +export function pointFromPercent( + frame: GestureReferenceFrame, + xPercent: number, + yPercent: number, + options: { marginPx?: number } = {}, +): GesturePoint { + const point = { + x: Math.round((frame.referenceWidth * xPercent) / 100), + y: Math.round((frame.referenceHeight * yPercent) / 100), + }; + if (options.marginPx === undefined) return point; + return clampGesturePoint(point, frame, options.marginPx); +} + +export function clampGesturePoint( + point: GesturePoint, + frame: GestureReferenceFrame, + marginPx: number, +): GesturePoint { + return { + x: clampGestureCoordinate(point.x, marginPx, frame.referenceWidth), + y: clampGestureCoordinate(point.y, marginPx, frame.referenceHeight), + }; +} + export function parseScrollDirection(direction: string): ScrollDirection { switch (direction) { case 'up': @@ -76,6 +128,19 @@ export function parseScrollDirection(direction: string): ScrollDirection { } } +function scrollDirectionForFingerSwipe(direction: ScrollDirection): ScrollDirection { + switch (direction) { + case 'up': + return 'down'; + case 'down': + return 'up'; + case 'left': + return 'right'; + case 'right': + return 'left'; + } +} + function resolveRequestedAmount(amount: number | undefined): number { if (amount === undefined) return DEFAULT_SCROLL_AMOUNT; if (!Number.isFinite(amount) || amount <= 0) { @@ -90,3 +155,8 @@ function normalizeRequestedPixels(pixels: number): number { } return Math.max(1, Math.round(pixels)); } + +export function clampGestureCoordinate(value: number, marginPx: number, size: number): number { + const max = Math.max(marginPx, size - marginPx); + return Math.min(max, Math.max(marginPx, value)); +} From 8e1dabcf1dc6eeeac23f8e3193df4c0fd9ddf8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 30 May 2026 13:52:51 +0200 Subject: [PATCH 2/2] fix: clarify Maestro gesture clamping --- .../__tests__/runtime-geometry.test.ts | 23 +++++++++++++++++++ src/compat/maestro/runtime-geometry.ts | 10 ++++---- src/core/__tests__/scroll-gesture.test.ts | 8 +++++++ src/core/scroll-gesture.ts | 9 ++++++-- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/compat/maestro/__tests__/runtime-geometry.test.ts b/src/compat/maestro/__tests__/runtime-geometry.test.ts index 32a53375b..0ef840fa2 100644 --- a/src/compat/maestro/__tests__/runtime-geometry.test.ts +++ b/src/compat/maestro/__tests__/runtime-geometry.test.ts @@ -61,3 +61,26 @@ test('swipeCoordinatesFromTarget preserves Maestro target-relative swipe distanc end: { x: 300, y: 240 }, }); }); + +test('swipeCoordinatesFromTarget clamps swipe endpoints to the viewport margin', () => { + const swipe = swipeCoordinatesFromTarget( + { + node: { + index: 12, + ref: 'e12', + type: 'Cell', + label: 'Card', + rect: { x: 340, y: 200, width: 100, height: 80 }, + }, + rect: { x: 340, y: 200, width: 100, height: 80 }, + frame: { referenceWidth: 402, referenceHeight: 874 }, + }, + 'right', + ); + + expect(swipe).toEqual({ + ok: true, + start: { x: 390, y: 240 }, + end: { x: 394, y: 240 }, + }); +}); diff --git a/src/compat/maestro/runtime-geometry.ts b/src/compat/maestro/runtime-geometry.ts index 83d2a8bbf..4d0875205 100644 --- a/src/compat/maestro/runtime-geometry.ts +++ b/src/compat/maestro/runtime-geometry.ts @@ -1,4 +1,4 @@ -import { clampGestureCoordinate } from '../../core/scroll-gesture.ts'; +import { clampToRange } from '../../core/scroll-gesture.ts'; import type { Rect, SnapshotNode } from '../../utils/snapshot.ts'; import { interiorCoordinate, pointInsideRect } from '../../utils/rect-center.ts'; import { normalizeType } from '../../utils/snapshot-processing.ts'; @@ -42,7 +42,7 @@ export function swipeCoordinatesFromTarget( start: center, end: { x: center.x, - y: clampGestureCoordinate(center.y - verticalDistance, minY, maxY + minY), + y: clampToRange(center.y - verticalDistance, minY, maxY), }, }; case 'down': @@ -51,7 +51,7 @@ export function swipeCoordinatesFromTarget( start: center, end: { x: center.x, - y: clampGestureCoordinate(center.y + verticalDistance, minY, maxY + minY), + y: clampToRange(center.y + verticalDistance, minY, maxY), }, }; case 'left': @@ -59,7 +59,7 @@ export function swipeCoordinatesFromTarget( ok: true, start: center, end: { - x: clampGestureCoordinate(center.x - horizontalDistance, minX, maxX + minX), + x: clampToRange(center.x - horizontalDistance, minX, maxX), y: center.y, }, }; @@ -68,7 +68,7 @@ export function swipeCoordinatesFromTarget( ok: true, start: center, end: { - x: clampGestureCoordinate(center.x + horizontalDistance, minX, maxX + minX), + x: clampToRange(center.x + horizontalDistance, minX, maxX), y: center.y, }, }; diff --git a/src/core/__tests__/scroll-gesture.test.ts b/src/core/__tests__/scroll-gesture.test.ts index 70ac2075b..de1485890 100644 --- a/src/core/__tests__/scroll-gesture.test.ts +++ b/src/core/__tests__/scroll-gesture.test.ts @@ -4,6 +4,7 @@ import { AppError } from '../../utils/errors.ts'; import { buildScrollGesturePlan, buildSwipeGesturePlan, + clampGestureCoordinate, pointFromPercent, } from '../scroll-gesture.ts'; @@ -86,3 +87,10 @@ test('pointFromPercent preserves unclamped percentages and clamps when a margin assert.deepEqual(pointFromPercent(frame, 125, -10), { x: 500, y: -80 }); assert.deepEqual(pointFromPercent(frame, 100, 0, { marginPx: 8 }), { x: 392, y: 8 }); }); + +test('clampGestureCoordinate rounds values and clamps them into the safe gesture band', () => { + assert.equal(clampGestureCoordinate(10.4, 8, 100), 10); + assert.equal(clampGestureCoordinate(10.6, 8, 100), 11); + assert.equal(clampGestureCoordinate(2.6, 8, 100), 8); + assert.equal(clampGestureCoordinate(97.6, 8, 100), 92); +}); diff --git a/src/core/scroll-gesture.ts b/src/core/scroll-gesture.ts index 6e35f2003..c64b32b0b 100644 --- a/src/core/scroll-gesture.ts +++ b/src/core/scroll-gesture.ts @@ -157,6 +157,11 @@ function normalizeRequestedPixels(pixels: number): number { } export function clampGestureCoordinate(value: number, marginPx: number, size: number): number { - const max = Math.max(marginPx, size - marginPx); - return Math.min(max, Math.max(marginPx, value)); + const min = marginPx; + const max = Math.max(min, size - marginPx); + return clampToRange(value, min, max); +} + +export function clampToRange(value: number, min: number, max: number): number { + return Math.min(Math.round(max), Math.max(Math.round(min), Math.round(value))); }