feat: add gesture telemetry recording overlays

ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -3,6 +3,16 @@ import XCTest
 extension RunnerTests {
   // MARK: - Main Thread Dispatch
 
+  private func currentUptimeMs() -> Double {
+    ProcessInfo.processInfo.systemUptime * 1000
+  }
+
+  private func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) {
+    let gestureStartUptimeMs = currentUptimeMs()
+    action()
+    return (gestureStartUptimeMs, currentUptimeMs())
+  }
+
   func execute(command: Command) throws -> Response {
     if Thread.isMainThread {
       return try executeOnMainSafely(command: command)
@@ -175,7 +185,7 @@ extension RunnerTests {
       }
       do {
         let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
-        let fpsLabel = command.fps.map(String.init) ?? "max"
+        let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps)
         NSLog(
           "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
           requestedOutPath,
@@ -204,26 +214,80 @@ extension RunnerTests {
         activeRecording = nil
         return Response(ok: false, error: ErrorPayload(message: "failed to stop recording: \(error.localizedDescription)"))
       }
+    case .uptime:
+      return Response(
+        ok: true,
+        data: DataPayload(currentUptimeMs: currentUptimeMs())
+      )
     case .tap:
       if let text = command.text {
         if let element = findElement(app: activeApp, text: text) {
-          element.tap()
-          return Response(ok: true, data: DataPayload(message: "tapped"))
+          let timing = measureGesture {
+            withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+              element.tap()
+            }
+          }
+          return Response(
+            ok: true,
+            data: DataPayload(
+              message: "tapped",
+              gestureStartUptimeMs: timing.gestureStartUptimeMs,
+              gestureEndUptimeMs: timing.gestureEndUptimeMs
+            )
+          )
         }
         return Response(ok: false, error: ErrorPayload(message: "element not found"))
       }
       if let x = command.x, let y = command.y {
-        tapAt(app: activeApp, x: x, y: y)
-        return Response(ok: true, data: DataPayload(message: "tapped"))
+        let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
+        let timing = measureGesture {
+          withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+            tapAt(app: activeApp, x: x, y: y)
+          }
+        }
+        return Response(
+          ok: true,
+          data: DataPayload(
+            message: "tapped",
+            gestureStartUptimeMs: timing.gestureStartUptimeMs,
+            gestureEndUptimeMs: timing.gestureEndUptimeMs,
+            x: touchFrame.x,
+            y: touchFrame.y,
+            referenceWidth: touchFrame.referenceWidth,
+            referenceHeight: touchFrame.referenceHeight
+          )
+        )
       }
       return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y"))
     case .mouseClick:
       guard let x = command.x, let y = command.y else {
         return Response(ok: false, error: ErrorPayload(message: "mouseClick requires x and y"))
       }
+      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
       do {
-        try mouseClickAt(app: activeApp, x: x, y: y, button: command.button ?? "primary")
-        return Response(ok: true, data: DataPayload(message: "clicked"))
+        var clickError: Error?
+        let timing = measureGesture {
+          do {
+            try mouseClickAt(app: activeApp, x: x, y: y, button: command.button ?? "primary")
+          } catch {
+            clickError = error
+          }
+        }
+        if let clickError {
+          throw clickError
+        }
+        return Response(
+          ok: true,
+          data: DataPayload(
+            message: "clicked",
+            gestureStartUptimeMs: timing.gestureStartUptimeMs,
+            gestureEndUptimeMs: timing.gestureEndUptimeMs,
+            x: touchFrame.x,
+            y: touchFrame.y,
+            referenceWidth: touchFrame.referenceWidth,
+            referenceHeight: touchFrame.referenceHeight
+          )
+        )
       } catch {
         return Response(ok: false, error: ErrorPayload(message: error.localizedDescription))
       }
@@ -234,32 +298,95 @@ extension RunnerTests {
       let count = max(Int(command.count ?? 1), 1)
       let intervalMs = max(command.intervalMs ?? 0, 0)
       let doubleTap = command.doubleTap ?? false
+      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
       if doubleTap {
-        runSeries(count: count, pauseMs: intervalMs) { _ in
-          doubleTapAt(app: activeApp, x: x, y: y)
+        let timing = measureGesture {
+          withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+            runSeries(count: count, pauseMs: intervalMs) { _ in
+              doubleTapAt(app: activeApp, x: x, y: y)
+            }
+          }
         }
-        return Response(ok: true, data: DataPayload(message: "tap series"))
+        return Response(
+          ok: true,
+          data: DataPayload(
+            message: "tap series",
+            gestureStartUptimeMs: timing.gestureStartUptimeMs,
+            gestureEndUptimeMs: timing.gestureEndUptimeMs,
+            x: touchFrame.x,
+            y: touchFrame.y,
+            referenceWidth: touchFrame.referenceWidth,
+            referenceHeight: touchFrame.referenceHeight
+          )
+        )
       }
-      runSeries(count: count, pauseMs: intervalMs) { _ in
-        tapAt(app: activeApp, x: x, y: y)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          runSeries(count: count, pauseMs: intervalMs) { _ in
+            tapAt(app: activeApp, x: x, y: y)
+          }
+        }
       }
-      return Response(ok: true, data: DataPayload(message: "tap series"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "tap series",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          x: touchFrame.x,
+          y: touchFrame.y,
+          referenceWidth: touchFrame.referenceWidth,
+          referenceHeight: touchFrame.referenceHeight
+        )
+      )
     case .longPress:
       guard let x = command.x, let y = command.y else {
         return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y"))
       }
       let duration = (command.durationMs ?? 800) / 1000.0
-      longPressAt(app: activeApp, x: x, y: y, duration: duration)
-      return Response(ok: true, data: DataPayload(message: "long pressed"))
+      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          longPressAt(app: activeApp, x: x, y: y, duration: duration)
+        }
+      }
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "long pressed",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          x: touchFrame.x,
+          y: touchFrame.y,
+          referenceWidth: touchFrame.referenceWidth,
+          referenceHeight: touchFrame.referenceHeight
+        )
+      )
     case .drag:
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
       }
       let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
-      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-        dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+      let dragFrame = resolvedDragVisualizationFrame(app: activeApp, x: x, y: y, x2: x2, y2: y2)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+        }
       }
-      return Response(ok: true, data: DataPayload(message: "dragged"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "dragged",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          x: dragFrame.x,
+          y: dragFrame.y,
+          x2: dragFrame.x2,
+          y2: dragFrame.y2,
+          referenceWidth: dragFrame.referenceWidth,
+          referenceHeight: dragFrame.referenceHeight
+        )
+      )
     case .dragSeries:
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries requires x, y, x2, and y2"))
@@ -271,17 +398,26 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
       }
       let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
-      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-        runSeries(count: count, pauseMs: pauseMs) { idx in
-          let reverse = pattern == "ping-pong" && (idx % 2 == 1)
-          if reverse {
-            dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
-          } else {
-            dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          runSeries(count: count, pauseMs: pauseMs) { idx in
+            let reverse = pattern == "ping-pong" && (idx % 2 == 1)
+            if reverse {
+              dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
+            } else {
+              dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+            }
           }
         }
       }
-      return Response(ok: true, data: DataPayload(message: "drag series"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "drag series",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      )
     case .type:
       guard let text = command.text else {
         return Response(ok: false, error: ErrorPayload(message: "type requires text"))
@@ -304,10 +440,22 @@ extension RunnerTests {
       guard let direction = command.direction else {
         return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
       }
-      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-        swipe(app: activeApp, direction: direction)
+      let referenceFrame = resolvedGestureReferenceFrame(app: activeApp)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          swipe(app: activeApp, direction: direction)
+        }
       }
-      return Response(ok: true, data: DataPayload(message: "swiped"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "swiped",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          referenceWidth: referenceFrame.referenceWidth,
+          referenceHeight: referenceFrame.referenceHeight
+        )
+      )
     case .findText:
       guard let text = command.text else {
         return Response(ok: false, error: ErrorPayload(message: "findText requires text"))
@@ -358,30 +506,15 @@ extension RunnerTests {
       if tapNavigationBack(app: activeApp) {
         return Response(ok: true, data: DataPayload(message: "back"))
       }
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "back button is not available on macOS"))
-#else
       performBackGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "back"))
-#endif
     case .home:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "home is not supported on macOS"))
-#else
       pressHomeButton()
       return Response(ok: true, data: DataPayload(message: "home"))
-#endif
     case .appSwitcher:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "appSwitcher is not supported on macOS"))
-#else
       performAppSwitcherGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "appSwitcher"))
-#endif
     case .alert:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "alert is not supported on macOS"))
-#else
       let action = (command.action ?? "get").lowercased()
       let alert = activeApp.alerts.firstMatch
       if !alert.exists {
@@ -399,17 +532,21 @@ extension RunnerTests {
       }
       let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label }
       return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels))
-#endif
     case .pinch:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "pinch is not supported on macOS"))
-#else
       guard let scale = command.scale, scale > 0 else {
         return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0"))
       }
-      pinch(app: activeApp, scale: scale, x: command.x, y: command.y)
-      return Response(ok: true, data: DataPayload(message: "pinched"))
-#endif
+      let timing = measureGesture {
+        pinch(app: activeApp, scale: scale, x: command.x, y: command.y)
+      }
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "pinched",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      )
     }
   }
 }