feat: support pixel-based scroll gestures

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -449,24 +449,45 @@ extension RunnerTests {
         typeIntoTarget(text)
       }
       return Response(ok: true, data: DataPayload(message: "typed"))
+    case .interactionFrame:
+      let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
+      return Response(
+        ok: true,
+        data: DataPayload(
+          x: frame.minX,
+          y: frame.minY,
+          referenceWidth: frame.width,
+          referenceHeight: frame.height
+        )
+      )
     case .swipe:
       guard let direction = command.direction else {
         return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
       }
-      let referenceFrame = resolvedGestureReferenceFrame(app: activeApp)
+      var executedFrame: DragVisualizationFrame?
       let timing = measureGesture {
         withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-          swipe(app: activeApp, direction: direction)
+          executedFrame = swipe(
+            app: activeApp,
+            direction: direction
+          )
         }
       }
+      guard let dragFrame = executedFrame else {
+        return Response(ok: false, error: ErrorPayload(message: "swipe is only supported on tvOS"))
+      }
       return Response(
         ok: true,
         data: DataPayload(
           message: "swiped",
           gestureStartUptimeMs: timing.gestureStartUptimeMs,
           gestureEndUptimeMs: timing.gestureEndUptimeMs,
-          referenceWidth: referenceFrame.referenceWidth,
-          referenceHeight: referenceFrame.referenceHeight
+          x: dragFrame.x,
+          y: dragFrame.y,
+          x2: dragFrame.x2,
+          y2: dragFrame.y2,
+          referenceWidth: dragFrame.referenceWidth,
+          referenceHeight: dragFrame.referenceHeight
         )
       )
     case .findText:

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -17,11 +17,6 @@ extension RunnerTests {
     let referenceHeight: Double
   }
 
-  struct GestureReferenceFrame {
-    let referenceWidth: Double
-    let referenceHeight: Double
-  }
-
   // MARK: - Navigation Gestures
 
   func tapInAppBackControl(app: XCUIApplication) -> Bool {
@@ -376,7 +371,7 @@ extension RunnerTests {
     )
   }
 
-  private func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
+  func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
     let window = app.windows.firstMatch
     let windowFrame = window.frame
     if window.exists && !windowFrame.isEmpty {
@@ -388,14 +383,6 @@ extension RunnerTests {
     return CGRect(x: 0, y: 0, width: 0, height: 0)
   }
 
-  func resolvedGestureReferenceFrame(app: XCUIApplication) -> GestureReferenceFrame {
-    let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
-    return GestureReferenceFrame(
-      referenceWidth: frame.width,
-      referenceHeight: frame.height
-    )
-  }
-
   func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
     let total = max(count, 1)
     let pause = max(pauseMs, 0)
@@ -407,39 +394,36 @@ extension RunnerTests {
     }
   }
 
-  func swipe(app: XCUIApplication, direction: SwipeDirection) {
+  func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {
-      return
-    }
-    let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
-    let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2))
-    let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.8))
-    let left = target.coordinate(withNormalizedOffset: CGVector(dx: 0.2, dy: 0.5))
-    let right = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
-
-    switch direction {
-    case .up:
-      end.press(forDuration: 0.1, thenDragTo: start)
-    case .down:
-      start.press(forDuration: 0.1, thenDragTo: end)
-    case .left:
-      right.press(forDuration: 0.1, thenDragTo: left)
-    case .right:
-      left.press(forDuration: 0.1, thenDragTo: right)
+      let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
+      let midX = frame.midX
+      let midY = frame.midY
+      return DragVisualizationFrame(
+        x: midX,
+        y: midY,
+        x2: midX,
+        y2: midY,
+        referenceWidth: frame.width,
+        referenceHeight: frame.height
+      )
     }
+    return nil
   }
 
-  private func performTvRemoteSwipeIfAvailable(direction: SwipeDirection) -> Bool {
+  private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool {
 #if os(tvOS)
     switch direction {
-    case .up:
+    case "up":
       XCUIRemote.shared.press(.up)
-    case .down:
+    case "down":
       XCUIRemote.shared.press(.down)
-    case .left:
+    case "left":
       XCUIRemote.shared.press(.left)
-    case .right:
+    case "right":
       XCUIRemote.shared.press(.right)
+    default:
+      return false
     }
     return true
 #else
@@ -515,5 +499,4 @@ extension RunnerTests {
     let element = app.descendants(matching: .any).matching(predicate).firstMatch
     return element.exists ? element : nil
   }
-
 }

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -152,7 +152,7 @@ extension RunnerTests {
 
   func isReadOnlyCommand(_ command: Command) -> Bool {
     switch command.command {
-    case .findText, .readText, .snapshot, .screenshot:
+    case .interactionFrame, .findText, .readText, .snapshot, .screenshot:
       return true
     case .alert:
       let action = (command.action ?? "get").lowercased()

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -5,6 +5,7 @@ enum CommandType: String, Codable {
   case mouseClick
   case tapSeries
   case longPress
+  case interactionFrame
   case drag
   case dragSeries
   case type
@@ -27,13 +28,6 @@ enum CommandType: String, Codable {
   case shutdown
 }
 
-enum SwipeDirection: String, Codable {
-  case up
-  case down
-  case left
-  case right
-}
-
 struct Command: Codable {
   let command: CommandType
   let appBundleId: String?
@@ -52,7 +46,7 @@ struct Command: Codable {
   let x2: Double?
   let y2: Double?
   let durationMs: Double?
-  let direction: SwipeDirection?
+  let direction: String?
   let scale: Double?
   let outPath: String?
   let fps: Int?

src/core/__tests__/dispatch-scroll.test.ts

@@ -0,0 +1,23 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { dispatchCommand } from '../dispatch.ts';
+import { AppError } from '../../utils/errors.ts';
+import type { DeviceInfo } from '../../utils/device.ts';
+
+const IOS_DEVICE: DeviceInfo = {
+  platform: 'ios',
+  id: 'sim-1',
+  name: 'iPhone 17 Pro',
+  kind: 'simulator',
+  booted: true,
+};
+
+test('dispatch scroll rejects mixing amount and --pixels', async () => {
+  await assert.rejects(
+    () => dispatchCommand(IOS_DEVICE, 'scroll', ['down', '0.4'], undefined, { pixels: 240 }),
+    (error: unknown) =>
+      error instanceof AppError &&
+      error.code === 'INVALID_ARGS' &&
+      /either a relative amount or --pixels/i.test(error.message),
+  );
+});

src/core/__tests__/scroll-gesture.test.ts

@@ -0,0 +1,56 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { AppError } from '../../utils/errors.ts';
+import { buildScrollGesturePlan } from '../scroll-gesture.ts';
+
+test('buildScrollGesturePlan maps relative amount to viewport travel', () => {
+  const plan = buildScrollGesturePlan({
+    direction: 'down',
+    amount: 0.5,
+    referenceWidth: 400,
+    referenceHeight: 800,
+  });
+
+  assert.deepEqual(plan, {
+    direction: 'down',
+    x1: 200,
+    y1: 600,
+    x2: 200,
+    y2: 200,
+    referenceWidth: 400,
+    referenceHeight: 800,
+    amount: 0.5,
+    pixels: 400,
+  });
+});
+
+test('buildScrollGesturePlan clamps pixel travel to the safe gesture band', () => {
+  const plan = buildScrollGesturePlan({
+    direction: 'right',
+    pixels: 500,
+    referenceWidth: 300,
+    referenceHeight: 600,
+  });
+
+  assert.equal(plan.x1, 285);
+  assert.equal(plan.x2, 15);
+  assert.equal(plan.y1, 300);
+  assert.equal(plan.y2, 300);
+  assert.equal(plan.pixels, 270);
+});
+
+test('buildScrollGesturePlan rejects invalid amounts', () => {
+  assert.throws(
+    () =>
+      buildScrollGesturePlan({
+        direction: 'down',
+        amount: 0,
+        referenceWidth: 400,
+        referenceHeight: 800,
+      }),
+    (error: unknown) =>
+      error instanceof AppError &&
+      error.code === 'INVALID_ARGS' &&
+      /amount must be a positive number/i.test(error.message),
+  );
+});

src/core/dispatch.ts

@@ -21,6 +21,7 @@ import type { RawSnapshotNode } from '../utils/snapshot.ts';
 import type { CliFlags } from '../utils/command-schema.ts';
 import { emitDiagnostic, withDiagnosticTimer } from '../utils/diagnostics.ts';
 import { successText, withSuccessText } from '../utils/success-text.ts';
+import { parseScrollDirection } from './scroll-gesture.ts';
 import {
   requireIntInRange,
   clampIosSwipeDuration,
@@ -67,6 +68,7 @@ export async function dispatchCommand(
     delayMs?: number;
     holdMs?: number;
     jitterPx?: number;
+    pixels?: number;
     doubleTap?: boolean;
     clickButton?: 'primary' | 'secondary' | 'middle';
     backMode?: 'in-app' | 'system';
@@ -381,13 +383,33 @@ export async function dispatchCommand(
           return { x, y, text, delayMs, ...successText(formatTextLengthMessage('Filled', text)) };
         }
         case 'scroll': {
-          const direction = positionals[0];
+          const directionInput = positionals[0];
           const amount = positionals[1] ? Number(positionals[1]) : undefined;
-          if (!direction) throw new AppError('INVALID_ARGS', 'scroll requires direction');
-          const interactionResult = await interactor.scroll(direction, amount);
+          const pixels = context?.pixels;
+          if (!directionInput) throw new AppError('INVALID_ARGS', 'scroll requires direction');
+          if (amount !== undefined && !Number.isFinite(amount)) {
+            throw new AppError('INVALID_ARGS', 'scroll amount must be a number');
+          }
+          if (amount !== undefined && pixels !== undefined) {
+            throw new AppError(
+              'INVALID_ARGS',
+              'scroll accepts either a relative amount or --pixels, not both',
+            );
+          }
+          const direction = parseScrollDirection(directionInput);
+          const interactionResult = await interactor.scroll(direction, { amount, pixels });
           return withSuccessText(
-            { direction, amount, ...interactionResult },
-            amount !== undefined ? `Scrolled ${direction} by ${amount}` : `Scrolled ${direction}`,
+            {
+              direction,
+              ...(amount !== undefined ? { amount } : {}),
+              ...(pixels !== undefined ? { pixels } : {}),
+              ...interactionResult,
+            },
+            pixels !== undefined
+              ? `Scrolled ${direction} by ${pixels}px`
+              : amount !== undefined
+                ? `Scrolled ${direction} by ${amount}`
+                : `Scrolled ${direction}`,
           );
         }
         case 'scrollintoview': {