From 5b7ccf7e5d839402739d71a07f9f923bf0fbbaff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 21 Mar 2026 17:48:21 +0100 Subject: [PATCH 01/11] feat: add gesture telemetry recording overlays --- .../RecordingScripts/recording-overlay.swift | 546 ++++++++++++++ .../RecordingScripts/recording-trim.swift | 137 ++++ .../__tests__/recording-gestures.test.ts | 187 +++++ src/daemon/__tests__/session-store.test.ts | 24 + .../handlers/__tests__/interaction.test.ts | 121 +++- .../handlers/__tests__/record-trace.test.ts | 297 +++++++- .../__tests__/session-replay-script.test.ts | 24 +- src/daemon/handlers/__tests__/session.test.ts | 3 + src/daemon/handlers/interaction.ts | 572 ++++++++++++++- src/daemon/handlers/record-trace-android.ts | 456 ++++++++++++ src/daemon/handlers/record-trace-recording.ts | 665 ++++++++++++++++++ src/daemon/handlers/record-trace.ts | 422 +---------- src/daemon/handlers/session-replay-script.ts | 38 + src/daemon/recording-gestures.ts | 539 ++++++++++++++ src/daemon/recording-telemetry.ts | 56 ++ src/daemon/recording-timing.ts | 37 + src/daemon/request-router.ts | 21 + src/daemon/session-store.ts | 20 + src/daemon/types.ts | 69 +- src/platforms/ios/recording-overlay.ts | 125 ++++ src/utils/__tests__/args.test.ts | 16 + src/utils/command-schema.ts | 12 +- src/utils/video.ts | 83 +++ 23 files changed, 3974 insertions(+), 496 deletions(-) create mode 100644 ios-runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift create mode 100644 ios-runner/AgentDeviceRunner/RecordingScripts/recording-trim.swift create mode 100644 src/daemon/__tests__/recording-gestures.test.ts create mode 100644 src/daemon/handlers/record-trace-android.ts create mode 100644 src/daemon/handlers/record-trace-recording.ts create mode 100644 src/daemon/recording-gestures.ts create mode 100644 src/daemon/recording-telemetry.ts create mode 100644 src/daemon/recording-timing.ts create mode 100644 src/platforms/ios/recording-overlay.ts create mode 100644 src/utils/video.ts diff --git a/ios-runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift b/ios-runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift new file mode 100644 index 000000000..698ecd07f --- /dev/null +++ b/ios-runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift @@ -0,0 +1,546 @@ +import AppKit +import AVFoundation +import Foundation +import QuartzCore + +let touchDotRadius: CGFloat = 30 +let touchDotColor = NSColor(calibratedRed: 0.20, green: 0.63, blue: 0.98, alpha: 0.48).cgColor +let touchDotBorderColor = NSColor(calibratedRed: 0.94, green: 0.98, blue: 1.0, alpha: 0.68).cgColor +let minimumTapVisibility: CFTimeInterval = 0.45 +let minimumSwipeVisibility: CFTimeInterval = 0.5 +let minimumPinchVisibility: CFTimeInterval = 0.5 +let swipeVisibilityTail: CFTimeInterval = 0.16 + +struct GestureEnvelope: Decodable { + let events: [GestureEvent] +} + +struct GestureEvent: Decodable { + let kind: String + let tMs: Double + let x: Double + let y: Double + let x2: Double? + let y2: Double? + let referenceWidth: Double? + let referenceHeight: Double? + let durationMs: Double? + let scale: Double? + let contentDirection: String? + let edge: String? +} + +enum OverlayError: Error, CustomStringConvertible { + case invalidArgs(String) + case missingVideoTrack + case exportFailed(String) + + var description: String { + switch self { + case .invalidArgs(let message): + return message + case .missingVideoTrack: + return "Input video does not contain a video track." + case .exportFailed(let message): + return message + } + } +} + +do { + try run() +} catch { + fputs("recording-overlay: \(error)\n", stderr) + exit(1) +} + +func run() throws { + let arguments = Array(CommandLine.arguments.dropFirst()) + let parsedArgs = try parseArguments(arguments) + let inputURL = URL(fileURLWithPath: parsedArgs.inputPath) + let outputURL = URL(fileURLWithPath: parsedArgs.outputPath) + let eventsURL = URL(fileURLWithPath: parsedArgs.eventsPath) + + if FileManager.default.fileExists(atPath: outputURL.path) { + try FileManager.default.removeItem(at: outputURL) + } + + let payload = try Data(contentsOf: eventsURL) + let envelope = try JSONDecoder().decode(GestureEnvelope.self, from: payload) + + if envelope.events.isEmpty { + try FileManager.default.copyItem(at: inputURL, to: outputURL) + return + } + + let asset = AVURLAsset(url: inputURL) + guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else { + throw OverlayError.missingVideoTrack + } + + let composition = AVMutableComposition() + guard let compositionVideoTrack = composition.addMutableTrack( + withMediaType: .video, + preferredTrackID: kCMPersistentTrackID_Invalid + ) else { + throw OverlayError.exportFailed("Failed to create composition video track.") + } + + let fullRange = CMTimeRange(start: .zero, duration: asset.duration) + try compositionVideoTrack.insertTimeRange(fullRange, of: sourceVideoTrack, at: .zero) + + if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first, + let compositionAudioTrack = composition.addMutableTrack( + withMediaType: .audio, + preferredTrackID: kCMPersistentTrackID_Invalid + ) { + try? compositionAudioTrack.insertTimeRange(fullRange, of: sourceAudioTrack, at: .zero) + } + + let renderSize = resolvedRenderSize(for: sourceVideoTrack) + let videoComposition = AVMutableVideoComposition() + videoComposition.renderSize = renderSize + videoComposition.frameDuration = resolvedFrameDuration(for: sourceVideoTrack) + + let instruction = AVMutableVideoCompositionInstruction() + instruction.timeRange = fullRange + let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack) + layerInstruction.setTransform(sourceVideoTrack.preferredTransform, at: .zero) + instruction.layerInstructions = [layerInstruction] + videoComposition.instructions = [instruction] + + let parentLayer = CALayer() + parentLayer.frame = CGRect(origin: .zero, size: renderSize) + parentLayer.masksToBounds = true + + let videoLayer = CALayer() + videoLayer.frame = parentLayer.frame + parentLayer.addSublayer(videoLayer) + + let overlayLayer = CALayer() + overlayLayer.frame = parentLayer.frame + parentLayer.addSublayer(overlayLayer) + + for event in envelope.events { + switch event.kind { + case "tap": + addTapLayer(event: event, renderSize: renderSize, to: overlayLayer) + case "longpress": + addLongPressLayer(event: event, renderSize: renderSize, to: overlayLayer) + case "swipe": + addSwipeLayers(event: event, renderSize: renderSize, to: overlayLayer) + case "scroll": + addScrollLayers(event: event, renderSize: renderSize, to: overlayLayer) + case "back-swipe": + addBackSwipeLayers(event: event, renderSize: renderSize, to: overlayLayer) + case "pinch": + addPinchLayers(event: event, renderSize: renderSize, to: overlayLayer) + default: + continue + } + } + + videoComposition.animationTool = AVVideoCompositionCoreAnimationTool( + postProcessingAsVideoLayer: videoLayer, + in: parentLayer + ) + + guard let exporter = AVAssetExportSession(asset: composition, presetName: AVAssetExportPresetHighestQuality) else { + throw OverlayError.exportFailed("Failed to create export session.") + } + + exporter.outputURL = outputURL + exporter.outputFileType = .mp4 + exporter.videoComposition = videoComposition + exporter.shouldOptimizeForNetworkUse = true + + let semaphore = DispatchSemaphore(value: 0) + exporter.exportAsynchronously { + semaphore.signal() + } + _ = semaphore.wait(timeout: .now() + 120) + + if exporter.status != .completed { + throw OverlayError.exportFailed(exporter.error?.localizedDescription ?? "Touch overlay export failed.") + } +} + +func parseArguments(_ arguments: [String]) throws -> (inputPath: String, outputPath: String, eventsPath: String) { + var inputPath: String? + var outputPath: String? + var eventsPath: String? + var index = 0 + + while index < arguments.count { + let argument = arguments[index] + let nextIndex = index + 1 + switch argument { + case "--input": + guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--input requires a value") } + inputPath = arguments[nextIndex] + index += 2 + case "--output": + guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--output requires a value") } + outputPath = arguments[nextIndex] + index += 2 + case "--events": + guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--events requires a value") } + eventsPath = arguments[nextIndex] + index += 2 + default: + throw OverlayError.invalidArgs("Unknown argument: \(argument)") + } + } + + guard let inputPath, let outputPath, let eventsPath else { + throw OverlayError.invalidArgs("Usage: recording-overlay.swift --input