diff --git a/android-snapshot-helper/README.md b/android-snapshot-helper/README.md index f5be62591..2808a6800 100644 --- a/android-snapshot-helper/README.md +++ b/android-snapshot-helper/README.md @@ -31,6 +31,7 @@ VERSION="$(node -p 'require("./package.json").version')" adb install -r -t ".tmp/android-snapshot-helper/agent-device-android-snapshot-helper-$VERSION.apk" adb shell am instrument -w \ -e waitForIdleTimeoutMs 500 \ + -e waitForIdleQuietMs 100 \ -e timeoutMs 8000 \ -e maxDepth 128 \ -e maxNodes 5000 \ @@ -59,6 +60,7 @@ The final instrumentation result includes: - `ok=true` - `helperApiVersion=1` - `waitForIdleTimeoutMs` +- `waitForIdleQuietMs` - `timeoutMs` - `maxDepth` - `maxNodes` diff --git a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 629b7291b..ad211a2ec 100644 --- a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -8,6 +8,10 @@ import android.util.Base64; import android.view.accessibility.AccessibilityNodeInfo; import android.view.accessibility.AccessibilityWindowInfo; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.lang.reflect.Field; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Locale; @@ -17,8 +21,9 @@ public final class SnapshotInstrumentation extends Instrumentation { private static final String PROTOCOL = "android-snapshot-helper-v1"; private static final String OUTPUT_FORMAT = "uiautomator-xml"; private static final String HELPER_API_VERSION = "1"; - private static final int CHUNK_SIZE = 8 * 1024; + private static final int CHUNK_SIZE = 2 * 1024; private static final long DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS = 500; + private static final long DEFAULT_WAIT_FOR_IDLE_QUIET_MS = 100; private static final long DEFAULT_TIMEOUT_MS = 8_000; private static final int DEFAULT_MAX_DEPTH = 128; private static final int DEFAULT_MAX_NODES = 5_000; @@ -36,21 +41,27 @@ public void onStart() { super.onStart(); long waitForIdleTimeoutMs = readLongArgument(arguments, "waitForIdleTimeoutMs", DEFAULT_WAIT_FOR_IDLE_TIMEOUT_MS); + long waitForIdleQuietMs = + readLongArgument(arguments, "waitForIdleQuietMs", DEFAULT_WAIT_FOR_IDLE_QUIET_MS); long timeoutMs = readLongArgument(arguments, "timeoutMs", DEFAULT_TIMEOUT_MS); int maxDepth = readIntArgument(arguments, "maxDepth", DEFAULT_MAX_DEPTH); int maxNodes = readIntArgument(arguments, "maxNodes", DEFAULT_MAX_NODES); + String outputPath = readStringArgument(arguments, "outputPath"); Bundle result = new Bundle(); result.putString("agentDeviceProtocol", PROTOCOL); result.putString("helperApiVersion", HELPER_API_VERSION); result.putString("outputFormat", OUTPUT_FORMAT); result.putString("waitForIdleTimeoutMs", Long.toString(waitForIdleTimeoutMs)); + result.putString("waitForIdleQuietMs", Long.toString(waitForIdleQuietMs)); result.putString("timeoutMs", Long.toString(timeoutMs)); result.putString("maxDepth", Integer.toString(maxDepth)); result.putString("maxNodes", Integer.toString(maxNodes)); try { long startedAtMs = System.currentTimeMillis(); - CaptureResult capture = captureXml(waitForIdleTimeoutMs, maxDepth, maxNodes); + CaptureResult capture = + captureXml(waitForIdleQuietMs, waitForIdleTimeoutMs, timeoutMs, maxDepth, maxNodes); + writeOutputFile(outputPath, capture.xml); emitChunks(capture.xml); result.putString("ok", "true"); result.putString("rootPresent", Boolean.toString(capture.rootPresent)); @@ -59,26 +70,111 @@ public void onStart() { result.putString("nodeCount", Integer.toString(capture.nodeCount)); result.putString("truncated", Boolean.toString(capture.truncated)); result.putString("elapsedMs", Long.toString(System.currentTimeMillis() - startedAtMs)); - finish(0, result); + finishSafely(0, result); } catch (Throwable error) { result.putString("ok", "false"); result.putString("errorType", error.getClass().getName()); result.putString( "message", error.getMessage() == null ? error.getClass().getName() : error.getMessage()); - finish(1, result); + finishSafely(1, result); + } + } + + private static String readStringArgument(Bundle arguments, String key) { + if (arguments == null || !arguments.containsKey(key)) { + return null; + } + String value = arguments.getString(key); + return value == null || value.trim().isEmpty() ? null : value.trim(); + } + + private static void writeOutputFile(String outputPath, String xml) throws IOException { + if (outputPath == null) { + return; + } + File file = new File(outputPath); + File parent = file.getParentFile(); + if (parent != null) { + parent.mkdirs(); + } + try (FileOutputStream stream = new FileOutputStream(file, false)) { + stream.write(xml.getBytes(StandardCharsets.UTF_8)); + } + } + + private void finishSafely(int resultCode, Bundle result) { + RuntimeException lastError = null; + for (int attempt = 0; attempt < 100; attempt += 1) { + try { + finish(resultCode, result); + return; + } catch (IllegalStateException error) { + if (!isUiAutomationConnectingError(error)) { + throw error; + } + lastError = error; + sleep(100); + } + } + detachUiAutomationBeforeFinish(); + try { + finish(resultCode, result); + return; + } catch (IllegalStateException error) { + if (!isUiAutomationConnectingError(error)) { + throw error; + } + lastError = error; + } + throw lastError; + } + + private void detachUiAutomationBeforeFinish() { + try { + Field field = Instrumentation.class.getDeclaredField("mUiAutomation"); + field.setAccessible(true); + field.set(this, null); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // If the platform blocks reflection, preserve the original finish failure below. + } + } + + private static boolean isUiAutomationConnectingError(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("while connecting"); + } + + private static boolean isUiAutomationNotConnectedError(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.toLowerCase(Locale.ROOT).contains("not connected"); + } + + private static void sleep(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); } } @SuppressWarnings("deprecation") - private CaptureResult captureXml(long waitForIdleTimeoutMs, int maxDepth, int maxNodes) + private CaptureResult captureXml( + long waitForIdleQuietMs, + long waitForIdleTimeoutMs, + long timeoutMs, + int maxDepth, + int maxNodes) throws TimeoutException { - UiAutomation automation = getUiAutomation(); + UiAutomation automation = getConnectedUiAutomation(timeoutMs); enableInteractiveWindowRetrieval(automation); if (waitForIdleTimeoutMs > 0) { try { - // Best-effort settle: avoids empty roots without inheriting UIAutomator's long idle wait. - automation.waitForIdle(waitForIdleTimeoutMs, waitForIdleTimeoutMs); + // Best-effort settle: wait for the accessibility stream to become idle, but require only + // a short quiet window once it does. Using the full timeout as the quiet window made every + // stable snapshot pay a fixed 500 ms tax. + long quietMs = Math.min(waitForIdleQuietMs, waitForIdleTimeoutMs); + automation.waitForIdle(quietMs, waitForIdleTimeoutMs); } catch (TimeoutException ignored) { // Busy or animated apps can still expose a usable root; capture whatever is available. } @@ -109,6 +205,30 @@ private CaptureResult captureXml(long waitForIdleTimeoutMs, int maxDepth, int ma xml.toString(), windowCount > 0, captureMode, windowCount, stats.nodeCount, stats.truncated); } + private UiAutomation getConnectedUiAutomation(long timeoutMs) throws TimeoutException { + long deadlineMs = System.currentTimeMillis() + Math.max(1, timeoutMs); + UiAutomation automation = getUiAutomation(); + RuntimeException lastError = null; + while (System.currentTimeMillis() <= deadlineMs) { + try { + automation.getServiceInfo(); + return automation; + } catch (IllegalStateException error) { + if (!isUiAutomationConnectingError(error) && !isUiAutomationNotConnectedError(error)) { + throw error; + } + lastError = error; + } + sleep(50); + } + TimeoutException timeout = + new TimeoutException("Timed out waiting for Android UiAutomation to connect"); + if (lastError != null) { + timeout.initCause(lastError); + } + throw timeout; + } + private static void enableInteractiveWindowRetrieval(UiAutomation automation) { AccessibilityServiceInfo serviceInfo; try { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index 17c1323da..0965ab9ce 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -252,7 +252,12 @@ extension RunnerTests { ) case .tap: if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { - let match = findElement(app: activeApp, selectorKey: selectorKey, selectorValue: selectorValue) + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true + ) if match.isAmbiguous { return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) } @@ -264,16 +269,24 @@ extension RunnerTests { var outcome = RunnerInteractionOutcome.performed let timing = measureGesture { withTemporaryScrollIdleTimeoutIfSupported(activeApp) { - outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + if match.usedNonHittableFallback { + // Maestro compatibility: RN E2E backdoor controls can be 1x1 and + // reported non-hittable by XCTest, while Maestro still taps their + // resolved bounds. Keep this behind the explicit replay-only flag. + outcome = tapAt(app: activeApp, x: frame.midX, y: frame.midY) + } else { + outcome = activateElement(app: activeApp, element: element, action: "tap by selector") + } } } if let response = unsupportedResponse(for: outcome) { return response } + waitForTextEntryReadinessAfterTap(app: activeApp, element: element) return Response( ok: true, data: DataPayload( - message: "tapped", + message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped", gestureStartUptimeMs: timing.gestureStartUptimeMs, gestureEndUptimeMs: timing.gestureEndUptimeMs, x: touchFrame?.x, @@ -729,6 +742,25 @@ extension RunnerTests { dismissed: result.dismissed ) ) + case .keyboardReturn: + let result = pressKeyboardReturn(app: activeApp) + if !result.pressed { + return Response( + ok: false, + error: ErrorPayload( + code: "UNSUPPORTED_OPERATION", + message: "Unable to press the iOS keyboard return key" + ) + ) + } + return Response( + ok: true, + data: DataPayload( + message: "keyboardReturn", + visible: result.visible, + wasVisible: result.wasVisible + ) + ) case .alert: let action = (command.action ?? "get").lowercased() guard let alert = resolveAlert(app: activeApp) else { @@ -839,7 +871,27 @@ extension RunnerTests { } let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0 let textEntryMode = resolveTextEntryMode(command) - let target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + let target: TextEntryTarget + if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue { + let match = findElement( + app: activeApp, + selectorKey: selectorKey, + selectorValue: selectorValue, + allowNonHittableFallback: command.allowNonHittableCoordinateFallback == true + ) + if match.isAmbiguous { + return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements")) + } + guard let element = match.element else { + return Response(ok: false, error: ErrorPayload(code: "NO_MATCH", message: "selector did not match an element")) + } + guard isTextEntryElement(element) else { + return Response(ok: false, error: ErrorPayload(code: "INVALID_TARGET", message: "selector did not match a text input")) + } + target = focusTextInputForTextEntry(app: activeApp, element: element) + } else { + target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y) + } if textEntryMode == .replacement { guard target.element != nil else { let message = @@ -867,6 +919,17 @@ extension RunnerTests { ) ) } - return Response(ok: true, data: DataPayload(message: textResult.repaired ? "typed after repair" : "typed")) + let point = target.refreshPoint + let frame = activeApp.frame + return Response( + ok: true, + data: DataPayload( + message: textResult.repaired ? "typed after repair" : "typed", + x: point.map { Double($0.x) }, + y: point.map { Double($0.y) }, + referenceWidth: frame.isEmpty ? nil : Double(frame.width), + referenceHeight: frame.isEmpty ? nil : Double(frame.height) + ) + ) } } diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index efeae75a9..cd231a953 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -27,6 +27,7 @@ extension RunnerTests { struct SelectorElementMatch { let element: XCUIElement? let isAmbiguous: Bool + let usedNonHittableFallback: Bool } enum TextTypingRepairMode { @@ -177,10 +178,15 @@ extension RunnerTests { return element.exists ? element : nil } - func findElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> SelectorElementMatch { + func findElement( + app: XCUIApplication, + selectorKey: String, + selectorValue: String, + allowNonHittableFallback: Bool = false + ) -> SelectorElementMatch { let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines) guard !value.isEmpty else { - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } let predicate: NSPredicate switch selectorKey { @@ -193,21 +199,47 @@ extension RunnerTests { case "text": predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value) default: - return SelectorElementMatch(element: nil, isAmbiguous: false) + return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false) } var matchedElement: XCUIElement? + var nonHittableElement: XCUIElement? let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex for element in matches where element.exists { - guard element.isHittable else { + if !element.isHittable { + if allowNonHittableFallback && hasTappableFrame(app: app, element: element) { + guard nonHittableElement == nil else { + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) + } + nonHittableElement = element + } continue } guard matchedElement == nil else { - return SelectorElementMatch(element: nil, isAmbiguous: true) + return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false) } matchedElement = element } - return SelectorElementMatch(element: matchedElement, isAmbiguous: false) + if let matchedElement { + return SelectorElementMatch(element: matchedElement, isAmbiguous: false, usedNonHittableFallback: false) + } + return SelectorElementMatch( + element: nonHittableElement, + isAmbiguous: false, + usedNonHittableFallback: nonHittableElement != nil + ) + } + + private func hasTappableFrame(app: XCUIApplication, element: XCUIElement) -> Bool { + let frame = element.frame + if frame.isEmpty { + return false + } + let appFrame = app.frame + if appFrame.isEmpty { + return true + } + return appFrame.contains(CGPoint(x: frame.midX, y: frame.midY)) } func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response { @@ -303,7 +335,7 @@ extension RunnerTests { switch element.elementType { case .textField, .secureTextField, .searchField, .textView: let frame = element.frame - return !frame.isEmpty && frame.contains(point) + return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2) default: return false } @@ -334,20 +366,33 @@ extension RunnerTests { return matched } + private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool { + point.x >= frame.minX - tolerance + && point.x <= frame.maxX + tolerance + && point.y >= frame.minY - tolerance + && point.y <= frame.maxY + tolerance + } + func focusedTextInput(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + // iOS focus predicates can return stale or misleading text-input matches + // under XCUITest, so text entry readiness is driven by tap/keyboard state. + return nil +#else var focused: XCUIElement? let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ - let candidate = app + let candidates = app .descendants(matching: .any) .matching(NSPredicate(format: "hasKeyboardFocus == 1")) - .firstMatch - guard candidate.exists else { return } - - switch candidate.elementType { - case .textField, .secureTextField, .searchField, .textView: - focused = candidate - default: - return + .allElementsBoundByIndex + for candidate in candidates where candidate.exists { + switch candidate.elementType { + case .textField, .secureTextField, .searchField, .textView: + focused = candidate + return + default: + continue + } } }) if let exceptionMessage { @@ -358,6 +403,7 @@ extension RunnerTests { return nil } return focused +#endif } func stabilizeTextInputBeforeTyping(app: XCUIApplication, target: XCUIElement?) -> XCUIElement? { @@ -417,6 +463,36 @@ extension RunnerTests { ) } + func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget { + let point = textEntryRefreshPoint(for: element) + if let point { + _ = tapAt(app: app, x: point.x, y: point.y) + } + let stabilized = stabilizeTextInputBeforeTyping(app: app, target: element) + let resolved = waitForTextEntryReadiness( + app: app, + target: TextEntryTarget( + element: stabilized ?? element, + refreshPoint: point, + prefersFocusedElement: false + ) + ) ?? stabilized ?? element + return TextEntryTarget( + element: resolved, + refreshPoint: textEntryRefreshPoint(for: resolved) ?? point, + prefersFocusedElement: false + ) + } + + func isTextEntryElement(_ element: XCUIElement) -> Bool { + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + func resolveTextEntryMode(_ command: Command) -> TextTypingRepairMode { switch command.textEntryMode { case "append": @@ -597,7 +673,7 @@ extension RunnerTests { guard let observedText = editableTextValue(for: targetElement) else { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } - guard observedText == expectedText else { + guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else { return TextEntryResult( verified: false, repaired: repaired, @@ -613,7 +689,11 @@ extension RunnerTests { return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil) } latestObservedText = nextObservedText - guard nextObservedText == expectedText else { + guard textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: nextObservedText, + expectedText: expectedText + ) else { return TextEntryResult( verified: false, repaired: repaired, @@ -630,6 +710,28 @@ extension RunnerTests { ) } + private func textEntryValueMatchesExpected( + _ element: XCUIElement?, + observedText: String, + expectedText: String + ) -> Bool { + if observedText == expectedText { + return true + } + guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else { + return false + } + var submittedText = expectedText + while hasTextEntrySubmitSuffix(submittedText) { + submittedText.removeLast() + } + return observedText == submittedText + } + + private func hasTextEntrySubmitSuffix(_ text: String) -> Bool { + text.hasSuffix("\n") || text.hasSuffix("\r") + } + private func expectedTextEntryValue( typedText: String, mode: TextTypingRepairMode, @@ -661,7 +763,11 @@ extension RunnerTests { guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else { return false } - if observedText == expectedText { + if textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: observedText, + expectedText: expectedText + ) { return false } latestObservedText = observedText @@ -678,7 +784,11 @@ extension RunnerTests { guard let latestObservedText else { return false } - guard latestObservedText != expectedText else { + guard !textEntryValueMatchesExpected( + resolveTextEntryElement(app: app, target: target), + observedText: latestObservedText, + expectedText: expectedText + ) else { return false } return isRepairableTextEntryMismatch( @@ -780,6 +890,35 @@ extension RunnerTests { #endif } + func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) { +#if os(iOS) + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil { + return + } + let frame = element.frame + if !frame.isEmpty { + _ = tapAt(app: app, x: frame.midX, y: frame.midY) + _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) + } + default: + return + } +#endif + } + + private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? { + let deadline = Date().addingTimeInterval(timeout) + while Date() < deadline { + if let focused = focusedTextInput(app: app) { + return focused + } + sleepFor(TextEntryTiming.pollInterval) + } + return focusedTextInput(app: app) + } + private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? { guard let element else { return nil @@ -843,6 +982,85 @@ extension RunnerTests { #endif } + func pressKeyboardReturn(app: XCUIApplication) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { +#if os(tvOS) + return (wasVisible: false, pressed: pressTvRemote(.select), visible: false) +#elseif os(iOS) + let wasVisible = isKeyboardVisible(app: app) + if tapKeyboardReturnControl(app: app) { + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + var typed = false + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + app.typeText(XCUIKeyboardKey.return.rawValue) + typed = true + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + if let singleTarget = singleTextEntryElement(app: app) { + return pressKeyboardReturn(on: singleTarget, app: app, wasVisible: wasVisible) + } + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: typed, visible: isKeyboardVisible(app: app)) +#else + return (wasVisible: false, pressed: false, visible: false) +#endif + } + + private func pressKeyboardReturn( + on element: XCUIElement, + app: XCUIApplication, + wasVisible: Bool + ) -> (wasVisible: Bool, pressed: Bool, visible: Bool) { + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + element.tap() + element.typeText(XCUIKeyboardKey.return.rawValue) + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TARGET_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return (wasVisible: wasVisible, pressed: false, visible: isKeyboardVisible(app: app)) + } + sleepFor(0.2) + return (wasVisible: wasVisible, pressed: true, visible: isKeyboardVisible(app: app)) + } + + private func singleTextEntryElement(app: XCUIApplication) -> XCUIElement? { +#if os(iOS) + var matches: [XCUIElement] = [] + let exceptionMessage = RunnerObjCExceptionCatcher.catchException({ + matches = app.descendants(matching: .any).allElementsBoundByIndex.filter { element in + guard element.exists else { return false } + switch element.elementType { + case .textField, .secureTextField, .searchField, .textView: + return true + default: + return false + } + } + }) + if let exceptionMessage { + NSLog( + "AGENT_DEVICE_RUNNER_KEYBOARD_RETURN_TEXT_ENTRY_QUERY_IGNORED_EXCEPTION=%@", + exceptionMessage + ) + return nil + } + return matches.count == 1 ? matches[0] : nil +#else + return nil +#endif + } + private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool { #if os(tvOS) return false @@ -880,6 +1098,22 @@ extension RunnerTests { #endif } + private func tapKeyboardReturnControl(app: XCUIApplication) -> Bool { +#if os(iOS) + for label in ["return", "Return", "Enter", "Go", "Search", "Next", "Done", "Send", "Join"] { + let candidates = [ + app.keyboards.buttons[label], + app.keyboards.keys[label], + ] + if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) { + hittable.tap() + return true + } + } +#endif + return false + } + private func isKeyboardAccessoryControl(_ element: XCUIElement, keyboardFrame: CGRect) -> Bool { let frame = element.frame guard !frame.isEmpty && !keyboardFrame.isEmpty else { @@ -942,11 +1176,24 @@ extension RunnerTests { guard !normalizedValue.isEmpty else { return false } - guard let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines), - !placeholder.isEmpty else { + let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if !placeholder.isEmpty && normalizedValue == placeholder { + return true + } + if isGenericTextInputLabel(normalizedValue) { + return true + } + let normalizedLabel = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + return normalizedLabel == normalizedValue && isGenericTextInputLabel(normalizedLabel) + } + + private func isGenericTextInputLabel(_ value: String) -> Bool { + switch value { + case "Text input field": + return true + default: return false } - return normalizedValue == placeholder } private func readableText(for element: XCUIElement) -> String? { diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index de5fa632f..5e60b4906 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -23,6 +23,7 @@ enum CommandType: String, Codable { case rotate case appSwitcher case keyboardDismiss + case keyboardReturn case alert case pinch case rotateGesture @@ -39,6 +40,7 @@ struct Command: Codable { let text: String? let selectorKey: String? let selectorValue: String? + let allowNonHittableCoordinateFallback: Bool? let delayMs: Int? let textEntryMode: String? let clearFirst: Bool? diff --git a/src/__tests__/client.test.ts b/src/__tests__/client.test.ts index 612c1b7b1..89791f066 100644 --- a/src/__tests__/client.test.ts +++ b/src/__tests__/client.test.ts @@ -253,6 +253,21 @@ test('replay.run keeps deprecated maestro option as backend alias', async () => assert.equal(setup.calls[0]?.flags?.replayBackend, 'maestro'); }); +test('replay.run forwards timeout budget', async () => { + const setup = createTransport(async () => ({ ok: true, data: {} })); + const client = createAgentDeviceClient(setup.config, { transport: setup.transport }); + + await client.replay.run({ + path: './flows/mod-lists.yaml', + backend: 'maestro', + timeoutMs: 240_000, + }); + + assert.equal(setup.calls.length, 1); + assert.equal(setup.calls[0]?.command, 'replay'); + assert.equal(setup.calls[0]?.flags?.timeoutMs, 240_000); +}); + test('client.command.wait prepares selector options and rejects invalid selectors', async () => { const setup = createTransport(async () => ({ ok: true, diff --git a/src/backend.ts b/src/backend.ts index eae5c551f..9cd46589d 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -92,7 +92,7 @@ export type BackendBackOptions = { }; export type BackendKeyboardOptions = { - action: 'status' | 'get' | 'dismiss'; + action: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type BackendKeyboardResult = { diff --git a/src/cli/commands/client-command.ts b/src/cli/commands/client-command.ts index 828846300..871c37da0 100644 --- a/src/cli/commands/client-command.ts +++ b/src/cli/commands/client-command.ts @@ -10,6 +10,7 @@ import type { import type { CliFlags } from '../../utils/command-schema.ts'; import { AppError } from '../../utils/errors.ts'; import { readCommandMessage } from '../../utils/success-text.ts'; +import { isKeyboardAction } from '../../utils/keyboard-actions.ts'; import { waitCommandCodec } from '../../command-codecs.ts'; import { parseDeviceRotation } from '../../core/device-rotation.ts'; import { buildSelectionOptions, writeCommandMessage, writeCommandOutput } from './shared.ts'; @@ -173,10 +174,13 @@ function readKeyboardAction( ): KeyboardCommandOptions['action'] | undefined { const action = value?.toLowerCase(); if (action === 'get') return 'status'; - if (action === undefined || action === 'status' || action === 'dismiss') { + if (action === undefined || (isKeyboardAction(action) && action !== 'get')) { return action; } - throw new AppError('INVALID_ARGS', 'keyboard action must be status, get, or dismiss.'); + throw new AppError( + 'INVALID_ARGS', + 'keyboard action must be status, get, dismiss, enter, or return.', + ); } function readFiniteNumber(value: string | undefined, label: string): number | undefined { diff --git a/src/cli/commands/generic.ts b/src/cli/commands/generic.ts index b75902085..e8c2be820 100644 --- a/src/cli/commands/generic.ts +++ b/src/cli/commands/generic.ts @@ -61,6 +61,7 @@ const genericClientCommandRunners = { update: flags.replayUpdate, backend: flags.replayMaestro ? 'maestro' : undefined, env: flags.replayEnv, + timeoutMs: flags.timeoutMs, }), test: ({ client, positionals, flags }) => { announceReplayTestRun({ json: flags.json }); @@ -68,6 +69,7 @@ const genericClientCommandRunners = { ...buildSelectionOptions(flags), paths: positionals, update: flags.replayUpdate, + backend: flags.replayMaestro ? 'maestro' : undefined, env: flags.replayEnv, failFast: flags.failFast, timeoutMs: flags.timeoutMs, diff --git a/src/client-types.ts b/src/client-types.ts index 2a44be877..dddbc47d9 100644 --- a/src/client-types.ts +++ b/src/client-types.ts @@ -389,7 +389,7 @@ export type RotateCommandOptions = DeviceCommandBaseOptions & { export type AppSwitcherCommandOptions = DeviceCommandBaseOptions; export type KeyboardCommandOptions = DeviceCommandBaseOptions & { - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter' | 'return'; }; export type ClipboardCommandOptions = @@ -449,7 +449,7 @@ export type AppSwitcherCommandResult = CommandActionResult<'app-switcher'>; export type KeyboardCommandResult = DaemonResponseData & { platform?: 'android' | 'ios'; - action?: 'status' | 'dismiss'; + action?: 'status' | 'dismiss' | 'enter'; visible?: boolean; inputType?: string | null; inputMethodPackage?: string | null; @@ -674,19 +674,24 @@ export type FindOptions = | (FindBaseOptions & { action: 'wait'; timeoutMs?: number }) | (FindBaseOptions & { action: 'fill' | 'type'; value: string }); -export type ReplayRunOptions = AgentDeviceRequestOverrides & { - path: string; - update?: boolean; - /** @deprecated Use backend: 'maestro'. */ - maestro?: boolean; - backend?: string; - env?: string[]; -}; +export type ReplayRunOptions = AgentDeviceRequestOverrides & + AgentDeviceSelectionOptions & { + path: string; + update?: boolean; + /** @deprecated Use backend: 'maestro'. */ + maestro?: boolean; + backend?: string; + env?: string[]; + timeoutMs?: number; + }; export type ReplayTestOptions = AgentDeviceRequestOverrides & AgentDeviceSelectionOptions & { paths: string[]; update?: boolean; + /** @deprecated Use backend: 'maestro'. */ + maestro?: boolean; + backend?: string; env?: string[]; failFast?: boolean; timeoutMs?: number; diff --git a/src/client.ts b/src/client.ts index 3d2f5e1ad..a9272b526 100644 --- a/src/client.ts +++ b/src/client.ts @@ -456,6 +456,7 @@ export function createAgentDeviceClient( await executeCommandRequest(PUBLIC_COMMANDS.test, options.paths, { ...options, replayUpdate: options.update, + replayBackend: options.backend ?? (options.maestro === true ? 'maestro' : undefined), replayEnv: options.env, replayShellEnv: collectReplayClientShellEnv(process.env), }), diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index b8443d27f..0940189d6 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -166,13 +166,7 @@ export const findCommand: RuntimeCommand = asyn disambiguateAmbiguous: false, }); if (!resolved) { - throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true })); + throw new AppError('COMMAND_FAILED', formatSelectorFailure(chain, [], { unique: true }), { + command: 'is', + reason: 'selector_not_found', + predicate: options.predicate, + selector: chain.raw, + }); } const result = evaluateIsPredicate({ predicate: options.predicate, @@ -316,6 +315,13 @@ export const isCommand: RuntimeCommand = asyn throw new AppError( 'COMMAND_FAILED', `is ${options.predicate} failed for selector ${resolved.selector.raw}: ${result.details}`, + { + command: 'is', + reason: 'predicate_failed', + predicate: options.predicate, + selector: resolved.selector.raw, + predicateDetails: result.details, + }, ); } return { @@ -400,19 +406,28 @@ async function waitForFindMatch( const timeout = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; const start = now(runtime); while (now(runtime) - start < timeout) { - const capture = await captureSelectorSnapshot(runtime, options, { - updateSession: true, - scope: shouldScopeFind(locator) ? options.query : undefined, - }); - const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { - requireRect: false, - }).matches[0]; + const { match } = await findFirstLocatorMatch(runtime, options, locator); if (match) return { kind: 'found', found: true, waitedMs: now(runtime) - start }; await sleep(runtime, POLL_INTERVAL_MS); } throw new AppError('COMMAND_FAILED', 'find wait timed out'); } +async function findFirstLocatorMatch( + runtime: AgentDeviceRuntime, + options: FindReadCommandOptions, + locator: FindLocator, +): Promise<{ capture: CapturedSnapshot; match: SnapshotNode | undefined }> { + const capture = await captureSelectorSnapshot(runtime, options, { + updateSession: true, + scope: shouldScopeFind(locator) ? options.query : undefined, + }); + const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { + requireRect: false, + }).matches[0]; + return { capture, match }; +} + async function waitForSelector( runtime: AgentDeviceRuntime, options: WaitCommandOptions, diff --git a/src/commands/system.ts b/src/commands/system.ts index adbfe7f01..9ab4ece5a 100644 --- a/src/commands/system.ts +++ b/src/commands/system.ts @@ -9,6 +9,7 @@ import type { CommandContext } from '../runtime-contract.ts'; import { AppError } from '../utils/errors.ts'; import { successText } from '../utils/success-text.ts'; import { requireIntInRange } from '../utils/validation.ts'; +import { isKeyboardAction } from '../utils/keyboard-actions.ts'; import type { RuntimeCommand } from './runtime-types.ts'; import { toBackendContext } from './selector-read-utils.ts'; import { normalizeOptionalText } from './text.ts'; @@ -44,7 +45,7 @@ export type SystemRotateCommandResult = { }; export type SystemKeyboardCommandOptions = CommandContext & { - action?: 'status' | 'get' | 'dismiss'; + action?: 'status' | 'get' | 'dismiss' | 'enter' | 'return'; }; export type SystemKeyboardCommandResult = @@ -60,6 +61,13 @@ export type SystemKeyboardCommandResult = state: BackendKeyboardResult; backendResult?: Record; message?: string; + } + | { + kind: 'keyboardEnterPressed'; + action: 'enter'; + state: BackendKeyboardResult; + backendResult?: Record; + message?: string; }; export type SystemClipboardCommandOptions = @@ -200,27 +208,22 @@ export const keyboardCommand: RuntimeCommand< throw new AppError('UNSUPPORTED_OPERATION', 'system.keyboard is not supported by this backend'); } const action = options.action ?? 'status'; - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'system.keyboard action must be status, get, or dismiss'); + if (!isKeyboardAction(action)) { + throw new AppError( + 'INVALID_ARGS', + 'system.keyboard action must be status, get, dismiss, enter, or return', + ); } const state = await runtime.backend.setKeyboard(toBackendContext(runtime, options), { action }); const formattedBackendResult = toBackendResult(state); + const keyboardState = isKeyboardResult(state) ? state : {}; + if (action === 'enter' || action === 'return') { + return normalizeKeyboardEnterResult(keyboardState, formattedBackendResult); + } if (action === 'dismiss') { - const dismissed = isKeyboardResult(state) ? state.dismissed : undefined; - return { - kind: 'keyboardDismissed', - action, - state: isKeyboardResult(state) ? state : {}, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - ...successText(dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), - }; + return normalizeKeyboardDismissResult(action, keyboardState, formattedBackendResult); } - return { - kind: 'keyboardState', - action, - state: isKeyboardResult(state) ? state : {}, - ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), - }; + return normalizeKeyboardStateResult(action, keyboardState, formattedBackendResult); }; export const clipboardCommand: RuntimeCommand< @@ -349,24 +352,79 @@ function normalizeAlertResult( result: BackendAlertResult, ): SystemAlertCommandResult { if (action === 'get') { - if (result.kind !== 'alertStatus') { - throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); - } - return { kind: 'alertStatus', action, alert: result.alert }; + return normalizeAlertStatusResult(result); } if (action === 'wait') { - if (result.kind !== 'alertWait') { - throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); - } - return { - kind: 'alertWait', - action, - alert: result.alert, - ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), - ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), - ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), - }; + return normalizeAlertWaitResult(result); + } + return normalizeAlertHandledResult(action, result); +} + +function normalizeKeyboardEnterResult( + state: BackendKeyboardResult, + backendResult: Record | undefined, +): SystemKeyboardCommandResult { + return { + kind: 'keyboardEnterPressed', + action: 'enter', + state, + ...(backendResult ? { backendResult } : {}), + ...successText('Keyboard enter pressed'), + }; +} + +function normalizeKeyboardDismissResult( + action: 'dismiss', + state: BackendKeyboardResult, + backendResult: Record | undefined, +): SystemKeyboardCommandResult { + return { + kind: 'keyboardDismissed', + action, + state, + ...(backendResult ? { backendResult } : {}), + ...successText(state.dismissed === false ? 'Keyboard already hidden' : 'Keyboard dismissed'), + }; +} + +function normalizeKeyboardStateResult( + action: 'status' | 'get', + state: BackendKeyboardResult, + backendResult: Record | undefined, +): SystemKeyboardCommandResult { + return { + kind: 'keyboardState', + action, + state, + ...(backendResult ? { backendResult } : {}), + }; +} + +function normalizeAlertStatusResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertStatus') { + throw new AppError('COMMAND_FAILED', 'system.alert get returned an invalid backend result'); + } + return { kind: 'alertStatus', action: 'get', alert: result.alert }; +} + +function normalizeAlertWaitResult(result: BackendAlertResult): SystemAlertCommandResult { + if (result.kind !== 'alertWait') { + throw new AppError('COMMAND_FAILED', 'system.alert wait returned an invalid backend result'); } + return { + kind: 'alertWait', + action: 'wait', + alert: result.alert, + ...(result.waitedMs !== undefined ? { waitedMs: result.waitedMs } : {}), + ...(result.timedOut !== undefined ? { timedOut: result.timedOut } : {}), + ...successText(result.alert ? 'Alert visible' : 'Alert wait timed out'), + }; +} + +function normalizeAlertHandledResult( + action: Exclude, + result: BackendAlertResult, +): SystemAlertCommandResult { if (result.kind !== 'alertHandled') { throw new AppError( 'COMMAND_FAILED', diff --git a/src/compat/__tests__/replay-input.test.ts b/src/compat/__tests__/replay-input.test.ts index 3b69320d5..d7d31fdfb 100644 --- a/src/compat/__tests__/replay-input.test.ts +++ b/src/compat/__tests__/replay-input.test.ts @@ -19,7 +19,7 @@ test('parseReplayInput routes compat replay scripts through the selected parser' parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="submit-order"']], + ['__maestroTapOn', ['id="submit-order"']], ], ); }); @@ -47,7 +47,7 @@ env: parsed.actions.map((action) => [action.command, action.positionals]), [ ['open', ['cli-app']], - ['click', ['id="shell-button"']], + ['__maestroTapOn', ['id="shell-button"']], ], ); }); diff --git a/src/compat/maestro/__tests__/replay-flow.test.ts b/src/compat/maestro/__tests__/replay-flow.test.ts index cf17308d5..72ab6ebe5 100644 --- a/src/compat/maestro/__tests__/replay-flow.test.ts +++ b/src/compat/maestro/__tests__/replay-flow.test.ts @@ -14,6 +14,9 @@ env: - launchApp - tapOn: id: home-open-form +- tapOn: + point: 20%,20% + label: Dismiss save password prompt - doubleTapOn: id: release-notice delay: 150 @@ -37,6 +40,11 @@ env: start: 50%, 75% end: 50%, 35% duration: 300 +- swipe: + direction: LEFT +- scrollUntilVisible: + element: Discover + direction: UP - takeScreenshot: ./screens/form.png - hideKeyboard - stopApp @@ -47,34 +55,239 @@ env: parsed.actions.map((entry) => [entry.command, entry.positionals]), [ ['open', ['com.callstack.agentdevicelab']], - ['click', ['id="home-open-form"']], + ['__maestroTapOn', ['id="home-open-form"']], + ['__maestroTapPointPercent', ['20', '20']], ['click', ['id="release-notice"']], - ['click', ['label="Agent Device Tester"']], + [ + 'click', + ['label="Agent Device Tester" || text="Agent Device Tester" || id="Agent Device Tester"'], + ], ['open', ['exp://localhost:8082']], - ['click', ['label="Full name" || text="Full name" || id="Full name"']], + ['__maestroTapOn', ['label="Full name" || text="Full name" || id="Full name"']], ['type', ['Ada Lovelace']], - ['wait', ['label="Checkout form"', '5000']], - ['is', ['hidden', 'label="Missing banner"']], - ['wait', ['id="submit-order"', '7000']], + [ + '__maestroAssertVisible', + ['label="Checkout form" || text="Checkout form" || id="Checkout form"', '5000'], + ], + [ + '__maestroAssertNotVisible', + ['label="Missing banner" || text="Missing banner" || id="Missing banner"'], + ], + ['__maestroAssertVisible', ['id="submit-order"', '7000']], ['scroll', ['down']], - ['scroll', ['down', '0.4']], + ['__maestroSwipeScreen', ['percent', '50', '75', '50', '35', '300']], + ['__maestroSwipeScreen', ['direction', 'left']], + [ + '__maestroScrollUntilVisible', + ['label="Discover" || text="Discover" || id="Discover"', '5000', 'up'], + ], ['screenshot', ['./screens/form.png']], ['keyboard', ['dismiss']], ['close', ['com.callstack.agentdevicelab']], ], ); - assert.equal(parsed.actions[2]?.flags.doubleTap, true); - assert.equal(parsed.actions[2]?.flags.intervalMs, 150); - assert.equal(parsed.actions[3]?.flags.holdMs, 3000); + assert.equal(parsed.actions[3]?.flags.doubleTap, true); + assert.equal(parsed.actions[3]?.flags.intervalMs, 150); + assert.equal(parsed.actions[4]?.flags.holdMs, 3000); + assert.equal(parsed.actions[1]?.flags.maestro?.allowNonHittableCoordinateFallback, true); + assert.equal(parsed.actions[6]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); +}); + +test('parseMaestroReplayFlow maps iOS openLink through the app id when available', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- openLink: exp://localhost:8082 +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['open', ['com.callstack.agentdevicelab', 'exp://localhost:8082']]], + ); +}); + +test('parseMaestroReplayFlow maps Android openLink like Maestro without package binding', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- openLink: exp://localhost:8082 +`, + { platform: 'android' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['open', ['exp://localhost:8082']]], + ); +}); + +test('parseMaestroReplayFlow converts Maestro nested selector compatibility syntax', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- eraseText +- eraseText: 12 +- tapOn: + id: childActionButton + childOf: + id: parent-row-secondary +- tapOn: + id: overflowButton + index: 0 +- tapOn: + label: Profile name metadata + text: Profile name +- swipe: + label: Drag item down + from: + id: reorder-handle + direction: UP + duration: 350 +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['\b'.repeat(50)]], + ['type', ['\b'.repeat(12)]], + [ + '__maestroTapOn', + ['id="childActionButton"', JSON.stringify({ childOf: 'id="parent-row-secondary"' })], + ], + ['__maestroTapOn', ['id="overflowButton"', JSON.stringify({ index: 0 })]], + ['__maestroTapOn', ['label="Profile name" || text="Profile name" || id="Profile name"']], + ['__maestroSwipeOn', ['id="reorder-handle"', 'up', '350']], + ], + ); +}); + +test('parseMaestroReplayFlow preserves runScript as an ordered runtime action', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-maestro-runscript-')); + const scriptPath = path.join(root, 'setup.js'); + const flowPath = path.join(root, 'flow.yml'); + fs.writeFileSync(scriptPath, `output.result = SERVER_PATH`); + + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runScript: + file: ./setup.js + env: + SERVER_PATH: local +- inputText: \${output.result} +`, + { sourcePath: flowPath }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroRunScript', [scriptPath]], + ['type', ['${output.result}']], + ], + ); + assert.deepEqual(parsed.actions[0]?.flags.maestro?.runScriptEnv, { SERVER_PATH: 'local' }); +}); + +test('parseMaestroReplayFlow keeps focused inputText and pressKey Enter as separate actions', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- inputText: hello +- pressKey: Enter +- inputText: world +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['type', ['hello']], + ['__maestroPressEnter', []], + ['type', ['world']], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 4, 5]); +}); + +test('parseMaestroReplayFlow marks tapOn before inputText for snapshot tap focus', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: editableNameInput +- inputText: Saved list +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroTapOn', ['id="editableNameInput"']], + ['type', ['Saved list']], + ], + ); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); +}); + +test('parseMaestroReplayFlow coalesces tapOn inputText while preserving pressKey Enter submit', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: + id: e2eProxyHeaderInput +- inputText: \${output.result} +- pressKey: Enter +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['wait', ['id="e2eProxyHeaderInput"', '30000']], + ['fill', ['id="e2eProxyHeaderInput"', '${output.result}']], + ['__maestroPressEnter', []], + ], + ); + assert.deepEqual(parsed.actionLines, [3, 3, 6]); + assert.equal(parsed.actions[1]?.flags?.maestro?.allowNonHittableCoordinateFallback, true); +}); + +test('parseMaestroReplayFlow does not coalesce text entry for non-input-looking targets', () => { + const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- tapOn: Continue +- inputText: unexpected +- pressKey: Enter +`); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [ + ['__maestroTapOn', ['label="Continue" || text="Continue" || id="Continue"']], + ['type', ['unexpected']], + ['__maestroPressEnter', []], + ], + ); + assert.equal(parsed.actions[0]?.flags?.maestro?.allowNonHittableCoordinateFallback, undefined); +}); + +test('parseMaestroReplayFlow rejects relative runScript paths without source path', () => { + assert.throws( + () => + parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- runScript: ./setup.js +`), + (error) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + /runScript file paths/.test(error.message), + ); }); test('parseMaestroReplayFlow rejects unsupported Maestro commands', () => { assert.throws( - () => parseMaestroReplayFlow('---\n- scrollUntilVisible: Save\n'), + () => parseMaestroReplayFlow('---\n- travelThroughTime: Save\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /issues\/558/.test(error.message) && /issues\/new/.test(error.message) && /line 2/.test(error.message), @@ -96,59 +309,14 @@ test('parseMaestroReplayFlow preserves selector state and absolute swipe command assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['wait', ['id="shipping-pickup" selected="true"', '5000']], + ['__maestroAssertVisible', ['id="shipping-pickup" selected="true"', '5000']], ['swipe', ['100', '500', '100', '200', '300']], ], ); assert.deepEqual(parsed.actionLines, [3, 6]); }); -test('parseMaestroReplayFlow maps easy Maestro device and utility commands', () => { - const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab -env: - VIDEO_PATH: ./recordings/checkout.mp4 ---- -- setAirplaneMode: true -- setAirplaneMode: false -- setLocation: - latitude: 52.2297 - longitude: 21.0122 -- setOrientation: landscapeLeft -- setPermissions: - camera: allow - microphone: deny - photos: unset - location: always -- killApp -- killApp: com.callstack.other -- pasteText: hello there -- startRecording: - path: \${VIDEO_PATH} -- stopRecording -- assertTrue: true -`); - - assert.deepEqual( - parsed.actions.map((entry) => [entry.command, entry.positionals]), - [ - ['settings', ['airplane', 'on']], - ['settings', ['airplane', 'off']], - ['settings', ['location', 'set', '52.2297', '21.0122']], - ['rotate', ['landscape-left']], - ['settings', ['permission', 'grant', 'camera']], - ['settings', ['permission', 'deny', 'microphone']], - ['settings', ['permission', 'reset', 'photos']], - ['settings', ['permission', 'grant', 'location-always']], - ['close', ['com.callstack.agentdevicelab']], - ['close', ['com.callstack.other']], - ['type', ['hello there']], - ['record', ['start', './recordings/checkout.mp4']], - ['record', ['stop']], - ], - ); -}); - -test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', () => { +test('parseMaestroReplayFlow rejects deferred Maestro utility commands loudly', () => { assert.throws( () => parseMaestroReplayFlow('---\n- assertTrue: "${READY}"\n'), (error) => @@ -160,11 +328,11 @@ test('parseMaestroReplayFlow rejects unsupported easy-mapping variants loudly', ); assert.throws( - () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: always\n'), + () => parseMaestroReplayFlow('---\n- setPermissions:\n camera: allow\n'), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /setPermissions state "always"/.test(error.message) && + /setPermissions/.test(error.message) && /issues\/558/.test(error.message) && /line 2/.test(error.message), ); @@ -196,12 +364,12 @@ test('parseMaestroReplayFlow reports top-level command lines around nested lists - runFlow: commands: - tapOn: Nested -- scrollUntilVisible: Save +- travelThroughTime: Save `), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /scrollUntilVisible/.test(error.message) && + /travelThroughTime/.test(error.message) && /line 6/.test(error.message), ); }); @@ -251,14 +419,14 @@ onFlowComplete: assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), [ - ['click', ['label="Before" || text="Before" || id="Before"']], - ['click', ['label="Nested" || text="Nested" || id="Nested"']], - ['click', ['id="child-repeat"']], - ['click', ['id="child-repeat"']], - ['click', ['label="iOS only" || text="iOS only" || id="iOS only"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="Again" || text="Again" || id="Again"']], - ['click', ['label="After" || text="After" || id="After"']], + ['__maestroTapOn', ['label="Before" || text="Before" || id="Before"']], + ['__maestroTapOn', ['label="Nested" || text="Nested" || id="Nested"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['id="child-repeat"']], + ['__maestroTapOn', ['label="iOS only" || text="iOS only" || id="iOS only"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="Again" || text="Again" || id="Again"']], + ['__maestroTapOn', ['label="After" || text="After" || id="After"']], ], ); }); @@ -279,57 +447,171 @@ test('parseMaestroReplayFlow skips platform-gated runFlow commands for other pla assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals]), - [['click', ['label="Shared" || text="Shared" || id="Shared"']]], + [['__maestroTapOn', ['label="Shared" || text="Shared" || id="Shared"']]], ); }); -test('parseMaestroReplayFlow tolerates false launchApp reset options and rejects reset side effects', () => { +test('parseMaestroReplayFlow treats Web platform gates as non-native branches', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + platform: Web + commands: + - tapOn: Web only +- tapOn: Native +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['__maestroTapOn', ['label="Native" || text="Native" || id="Native"']]], + ); +}); + +test('parseMaestroReplayFlow evaluates simple runFlow.when.true platform expressions', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + true: \${maestro.platform == 'android' || maestro.platform == 'ios'} + commands: + - tapOn: Native +- runFlow: + when: + true: \${maestro.platform == 'web' || maestro.platform == 'android'} + commands: + - tapOn: Not iOS +`, + { platform: 'ios' }, + ); + + assert.deepEqual( + parsed.actions.map((entry) => [entry.command, entry.positionals]), + [['__maestroTapOn', ['label="Native" || text="Native" || id="Native"']]], + ); +}); + +test('parseMaestroReplayFlow keeps visible-gated runFlow commands for runtime evaluation', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- runFlow: + when: + visible: Continue + commands: + - tapOn: Continue +`, + { platform: 'ios' }, + ); + + assert.equal(parsed.actions[0]?.command, '__maestroRunFlowWhen'); + assert.deepEqual(parsed.actions[0]?.positionals, [ + 'visible', + 'label="Continue" || text="Continue" || id="Continue"', + ]); + assert.deepEqual(parsed.actions[0]?.flags.batchSteps, [ + { + command: '__maestroTapOn', + positionals: ['label="Continue" || text="Continue" || id="Continue"'], + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, + }, + ]); +}); + +test('parseMaestroReplayFlow keeps retry commands for runtime evaluation', () => { + const parsed = parseMaestroReplayFlow( + `appId: com.callstack.agentdevicelab +--- +- retry: + maxRetries: 3 + commands: + - openLink: + link: \${APP_SCHEME}details + - assertVisible: Article +`, + { env: { APP_SCHEME: 'example://' } }, + ); + + assert.equal(parsed.actions[0]?.command, '__maestroRetry'); + assert.deepEqual(parsed.actions[0]?.positionals, ['3']); + assert.deepEqual(parsed.actions[0]?.flags.batchSteps, [ + { + command: 'open', + positionals: ['example://details'], + flags: {}, + }, + { + command: '__maestroAssertVisible', + positionals: ['label="Article" || text="Article" || id="Article"', '5000'], + flags: {}, + }, + ]); +}); + +test('parseMaestroReplayFlow accepts launchApp reset options', () => { const parsed = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: - clearState: false - clearKeychain: false + clearState: true + arguments: + "-EXDevMenuIsOnboardingFinished": true + launchArguments: + "-Example": "ignored" stopApp: true `); assert.deepEqual( parsed.actions.map((entry) => [entry.command, entry.positionals, entry.flags]), - [['open', ['com.callstack.agentdevicelab'], { relaunch: true }]], + [ + [ + 'open', + ['com.callstack.agentdevicelab'], + { + clearAppState: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true', '-Example', 'ignored'], + }, + ], + ], ); +}); +test('parseMaestroReplayFlow rejects clearKeychain instead of ignoring it', () => { assert.throws( () => parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- - launchApp: - clearState: true + clearKeychain: true `), (error) => error instanceof AppError && error.code === 'INVALID_ARGS' && - /clearState: true/.test(error.message) && - /line 3/.test(error.message), + /clearKeychain/.test(error.message), ); }); -test('parseMaestroReplayFlow rejects runtime-dependent flow control for now', () => { - assert.throws( - () => - parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +test('parseMaestroReplayFlow relaunches launchApp only when clearState is absent', () => { + const withLaunchArgs = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab --- -- runFlow: - when: - visible: Continue - commands: - - tapOn: Continue -`), - (error) => - error instanceof AppError && - error.code === 'INVALID_ARGS' && - /when.visible/.test(error.message) && - /line 3/.test(error.message), - ); +- launchApp: + arguments: + "-Example": "value" +`); + const withStopApp = parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab +--- +- launchApp: + stopApp: true +`); + + assert.equal(withLaunchArgs.actions[0]?.flags.relaunch, true); + assert.equal(withStopApp.actions[0]?.flags.relaunch, true); +}); +test('parseMaestroReplayFlow rejects unsupported runtime-dependent flow control', () => { assert.throws( () => parseMaestroReplayFlow(`appId: com.callstack.agentdevicelab @@ -359,24 +641,24 @@ test('parseMaestroReplayFlow parses the test-app Maestro suite fixture', () => { assert.deepEqual( parsed.actions.map((entry) => entry.command), [ - 'wait', - 'click', - 'wait', - 'click', + '__maestroAssertVisible', + '__maestroTapOn', + '__maestroAssertVisible', + '__maestroTapOn', 'type', - 'click', + '__maestroTapOn', 'type', - 'click', - 'wait', - 'wait', - 'scroll', - 'click', - 'wait', - 'click', - 'wait', - 'click', - 'wait', - 'wait', + '__maestroTapOn', + '__maestroAssertVisible', + '__maestroAssertVisible', + '__maestroSwipeScreen', + '__maestroTapOn', + '__maestroAssertVisible', + '__maestroTapOn', + '__maestroAssertVisible', + '__maestroTapOn', + '__maestroAssertVisible', + '__maestroAssertVisible', ], ); }); diff --git a/src/compat/maestro/__tests__/runtime-targets.test.ts b/src/compat/maestro/__tests__/runtime-targets.test.ts new file mode 100644 index 000000000..b77ff7aa0 --- /dev/null +++ b/src/compat/maestro/__tests__/runtime-targets.test.ts @@ -0,0 +1,202 @@ +import { test, expect } from 'vitest'; +import type { SnapshotState } from '../../../utils/snapshot.ts'; +import { + resolveMaestroNodeFromSnapshot, + resolveVisibleMaestroNodeFromSnapshot, +} from '../runtime-targets.ts'; + +test('resolveVisibleMaestroNodeFromSnapshot treats app content behind React Native overlays as hidden', () => { + const snapshot = makeReactNativeOverlaySnapshot(); + + const appContent = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Article title" || text="Article title" || id="Article title"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + const overlayControl = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Minimize" || text="Minimize" || id="Minimize"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(appContent).toMatchObject({ + ok: false, + message: expect.stringContaining('React Native overlay is covering app content'), + }); + expect(overlayControl).toMatchObject({ + ok: true, + node: expect.objectContaining({ label: 'Minimize' }), + }); +}); + +test('resolveMaestroNodeFromSnapshot blocks taps on app content behind React Native overlays', () => { + const snapshot = makeReactNativeOverlaySnapshot(); + + const appContent = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Article title" || text="Article title" || id="Article title"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + const overlayControl = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Dismiss" || text="Dismiss" || id="Dismiss"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(appContent).toMatchObject({ + ok: false, + message: expect.stringContaining('React Native overlay is covering app content'), + }); + expect(overlayControl).toMatchObject({ + ok: true, + node: expect.objectContaining({ label: 'Dismiss' }), + }); +}); + +test('resolveMaestroNodeFromSnapshot prefers foreground duplicate matches', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'button', + label: 'Show Dialog', + rect: { x: 24, y: 220, width: 240, height: 72 }, + depth: 8, + }, + { + index: 2, + ref: 'e2', + type: 'button', + label: 'Show Dialog', + rect: { x: 24, y: 220, width: 240, height: 72 }, + depth: 8, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Show Dialog" || text="Show Dialog" || id="Show Dialog"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 2 }), + }); +}); + +test('resolveMaestroNodeFromSnapshot preserves read order for duplicate matches in different rects', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'button', + label: 'Open details', + rect: { x: 24, y: 520, width: 240, height: 72 }, + depth: 8, + }, + { + index: 2, + ref: 'e2', + type: 'button', + label: 'Open details', + rect: { x: 24, y: 320, width: 240, height: 72 }, + depth: 8, + }, + ], + }; + + const target = resolveMaestroNodeFromSnapshot( + snapshot, + 'label="Open details" || text="Open details" || id="Open details"', + {}, + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(target).toMatchObject({ + ok: true, + node: expect.objectContaining({ index: 1 }), + }); +}); + +test('resolveVisibleMaestroNodeFromSnapshot requires visible text matches to be on screen', () => { + const snapshot: SnapshotState = { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.TextView', + label: 'Library', + rect: { x: 0, y: 2340, width: 120, height: 48 }, + depth: 8, + }, + ], + }; + + const target = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + 'label="Library" || text="Library" || id="Library"', + 'android', + { referenceWidth: 1080, referenceHeight: 2340 }, + ); + + expect(target).toMatchObject({ + ok: false, + message: expect.stringContaining('none were visible'), + }); +}); + +function makeReactNativeOverlaySnapshot(): SnapshotState { + return { + createdAt: Date.now(), + nodes: [ + { + index: 1, + ref: 'e1', + type: 'android.widget.TextView', + label: 'Article title', + rect: { x: 24, y: 420, width: 320, height: 54 }, + depth: 8, + }, + { + index: 2, + ref: 'e2', + type: 'android.widget.TextView', + label: 'AppStack.tsx (42:7)', + rect: { x: 28, y: 1304, width: 1025, height: 44 }, + depth: 8, + }, + { + index: 3, + ref: 'e3', + type: 'android.view.ViewGroup', + label: 'Dismiss', + rect: { x: 0, y: 2142, width: 540, height: 132 }, + depth: 6, + }, + { + index: 4, + ref: 'e4', + type: 'android.view.ViewGroup', + label: 'Minimize', + rect: { x: 540, y: 2142, width: 540, height: 132 }, + depth: 6, + }, + ], + }; +} diff --git a/src/compat/maestro/command-mapper.ts b/src/compat/maestro/command-mapper.ts index 76a3bc1b7..36f294b40 100644 --- a/src/compat/maestro/command-mapper.ts +++ b/src/compat/maestro/command-mapper.ts @@ -1,23 +1,14 @@ import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; -import { - convertAssertTrue, - convertKillApp, - convertLaunchApp, - convertSetAirplaneMode, - convertSetLocation, - convertSetOrientation, - convertSetPermissions, - convertStartRecording, - convertStopApp, - convertStopRecording, -} from './device-actions.ts'; +import { convertLaunchApp, convertStopApp } from './device-actions.ts'; import { convertDoubleTapOn, + convertEraseText, convertExtendedWaitUntil, convertLongPressOn, convertPressKey, convertScroll, + convertScrollUntilVisible, convertSwipe, convertTapOn, maestroSelector, @@ -27,15 +18,15 @@ import { action, assertOnlyKeys, isPlainRecord, - normalizeCommandList, - normalizePlatformValue, - readEnvMap, readTimeoutMs, + requireAppId, requireStringValue, resolveMaestroString, unsupportedCommand, - unsupportedMaestroSyntax, } from './support.ts'; +import { convertRepeat, convertRetry, convertRunFlow } from './flow-control.ts'; +import { convertRunScript } from './run-script.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroCommand, MaestroCommandMapperDeps, @@ -43,7 +34,6 @@ import type { MaestroParseContext, } from './types.ts'; -const MAX_REPEAT_EXPANSIONS = 100; type MaestroCommandHandler = (params: { value: unknown; config: MaestroFlowConfig; @@ -60,39 +50,41 @@ const MAP_COMMAND_HANDLERS: Record = { inputText: ({ value, context }) => [ action('type', [resolveMaestroString(readInputText(value), context)]), ], + eraseText: ({ value }) => [convertEraseText(value)], pasteText: ({ value, context, name }) => [ action('type', [resolveMaestroString(requireStringValue(name, value), context)]), ], - openLink: ({ value, context, name }) => [ - action('open', [resolveMaestroString(requireStringValue(name, value), context)]), - ], + openLink: ({ value, config, context, name }) => [convertOpenLink(value, config, context, name)], assertVisible: ({ value, context, name }) => [ - action('wait', [maestroSelector(value, name, [], context), '5000']), + action(MAESTRO_RUNTIME_COMMAND.assertVisible, [ + maestroSelector(value, name, [], context), + '5000', + ]), ], assertNotVisible: ({ value, context, name }) => [ - action('is', ['hidden', maestroSelector(value, name, [], context)]), + action(MAESTRO_RUNTIME_COMMAND.assertNotVisible, [maestroSelector(value, name, [], context)]), ], - assertTrue: ({ value, context }) => convertAssertTrue(value, context), extendedWaitUntil: ({ value, context }) => convertExtendedWaitUntil(value, context), takeScreenshot: ({ value, context, name }) => [ action('screenshot', [resolveMaestroString(requireStringValue(name, value), context)]), ], scroll: ({ value }) => [convertScroll(value)], - swipe: ({ value }) => [convertSwipe(value)], + scrollUntilVisible: ({ value, context }) => convertScrollUntilVisible(value, context), + swipe: ({ value, context }) => [convertSwipe(value, context)], hideKeyboard: () => [action('keyboard', ['dismiss'])], pressKey: ({ value }) => [convertPressKey(value)], back: () => [action('back')], - waitForAnimationToEnd: ({ value }) => [action('wait', [String(readTimeoutMs(value, 250))])], + waitForAnimationToEnd: ({ value }) => [ + action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, [String(readTimeoutMs(value, 15000))]), + ], stopApp: ({ value, config, context }) => [convertStopApp(value, config, context)], - killApp: ({ value, config, context }) => [convertKillApp(value, config, context)], - setAirplaneMode: ({ value, context }) => [convertSetAirplaneMode(value, context)], - setLocation: ({ value, context }) => [convertSetLocation(value, context)], - setOrientation: ({ value, context }) => [convertSetOrientation(value, context)], - setPermissions: ({ value, context }) => convertSetPermissions(value, context), - startRecording: ({ value, context }) => [convertStartRecording(value, context)], - stopRecording: ({ value }) => [convertStopRecording(value)], - runFlow: ({ value, config, context, deps }) => convertRunFlow(value, config, context, deps), - repeat: ({ value, config, context, deps }) => convertRepeat(value, config, context, deps), + runScript: ({ value, context }) => [convertRunScript(value, context)], + runFlow: ({ value, config, context, deps }) => + convertRunFlow(value, config, context, deps, convertCommandList), + repeat: ({ value, config, context, deps }) => + convertRepeat(value, config, context, deps, convertCommandList), + retry: ({ value, config, context, deps }) => + convertRetry(value, config, context, deps, convertCommandList), }; const SCALAR_COMMAND_HANDLERS: Record< @@ -102,12 +94,10 @@ const SCALAR_COMMAND_HANDLERS: Record< launchApp: (config, context) => [convertLaunchApp(undefined, config, context)], scroll: () => [action('scroll', ['down'])], hideKeyboard: () => [action('keyboard', ['dismiss'])], + eraseText: () => [convertEraseText(undefined)], back: () => [action('back')], - waitForAnimationToEnd: () => [action('wait', ['250'])], + waitForAnimationToEnd: () => [action(MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd, ['15000'])], stopApp: (config, context) => [convertStopApp(undefined, config, context)], - killApp: (config, context) => [convertKillApp(undefined, config, context)], - startRecording: () => [action('record', ['start'])], - stopRecording: () => [action('record', ['stop'])], }; export function convertMaestroCommandWithLine( @@ -156,63 +146,25 @@ function convertScalarCommand( return handler(config, context); } -function convertRunFlow( +function convertOpenLink( value: unknown, config: MaestroFlowConfig, context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (typeof value === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; - } - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); - } - assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); - if (!shouldRunFlow(value.when, context)) return []; - - const runContext = { - ...context, - env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, - }; - if (typeof value.file === 'string') { - return deps.parseRunFlowFile(resolveMaestroString(value.file, runContext), runContext).actions; + name: string, +): SessionAction { + const rawLink = readOpenLink(value, name); + const url = resolveMaestroString(rawLink, context); + if (context.platform === 'ios' && config.appId) { + return action('open', [resolveMaestroString(requireAppId(config, name), context), url]); } - if (Array.isArray(value.commands)) { - return convertCommandList(normalizeCommandList(value.commands), config, runContext, deps); - } - throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); + return action('open', [url]); } -function convertRepeat( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, - deps: MaestroCommandMapperDeps, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'repeat expects a map.'); - } - assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); - if (value.while !== undefined) { - throw unsupportedMaestroSyntax( - 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', - ); - } - const times = readRepeatTimes(value.times, context); - if (!Array.isArray(value.commands)) { - throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); - } - if (times > MAX_REPEAT_EXPANSIONS) { - throw new AppError( - 'INVALID_ARGS', - `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, - ); - } - const commands = normalizeCommandList(value.commands); - return Array.from({ length: times }).flatMap(() => - convertCommandList(commands, config, context, deps), - ); +function readOpenLink(value: unknown, name: string): string { + if (typeof value === 'string') return value; + if (!isPlainRecord(value)) return requireStringValue(name, value); + assertOnlyKeys(value, name, ['link']); + return requireStringValue(`${name}.link`, value.link); } function convertCommandList( @@ -225,50 +177,3 @@ function convertCommandList( convertMaestroCommandWithLine(command, config, index + 1, context, deps), ); } - -function shouldRunFlow(value: unknown, context: MaestroParseContext): boolean { - if (value === undefined || value === null) return true; - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); - } - assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); - rejectUnsupportedCondition(value, 'visible', 'when.visible'); - rejectUnsupportedCondition(value, 'notVisible', 'when.notVisible'); - rejectUnsupportedCondition(value, 'true', 'when.true'); - if (value.platform === undefined) return true; - const platform = normalizePlatformValue(value.platform, 'runFlow.when.platform'); - if (!context.platform) { - throw new AppError( - 'INVALID_ARGS', - 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', - ); - } - return platform === context.platform; -} - -function readRepeatTimes(value: unknown, context: MaestroParseContext): number { - const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && /^\d+$/.test(resolved) - ? Number(resolved) - : undefined; - if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { - throw new AppError( - 'INVALID_ARGS', - 'repeat.times must be a non-negative integer or ${VAR} resolving to one.', - ); - } - return numeric; -} - -function rejectUnsupportedCondition( - value: Record, - key: string, - label: string, -): void { - if (value[key] !== undefined) { - throw unsupportedMaestroSyntax(`Maestro ${label} is not supported yet.`); - } -} diff --git a/src/compat/maestro/device-actions.ts b/src/compat/maestro/device-actions.ts index 95e33db4d..433629b0e 100644 --- a/src/compat/maestro/device-actions.ts +++ b/src/compat/maestro/device-actions.ts @@ -4,50 +4,11 @@ import { action, assertOnlyKeys, isPlainRecord, - normalizeToken, - readBooleanLiteral, requireAppId, resolveMaestroString, - resolveMaybeMaestroString, unsupportedMaestroSyntax, } from './support.ts'; -import type { MaestroFlowConfig, MaestroParseContext, PermissionCommand } from './types.ts'; - -const SUPPORTED_PERMISSION_TARGETS = new Set([ - 'accessibility', - 'calendar', - 'camera', - 'contacts', - 'contacts-limited', - 'input-monitoring', - 'location', - 'location-always', - 'media-library', - 'microphone', - 'motion', - 'notifications', - 'photos', - 'reminders', - 'screen-recording', - 'siri', -]); - -const BASIC_PERMISSION_STATES: Record = { - allow: 'grant', - grant: 'grant', - granted: 'grant', - deny: 'deny', - denied: 'deny', - reset: 'reset', - unset: 'reset', - revoke: 'reset', - revoked: 'reset', -}; - -const MODE_PERMISSION_STATES: Record = { - limited: { command: 'grant', mode: 'limited' }, - full: { command: 'grant', mode: 'full' }, -}; +import type { MaestroFlowConfig, MaestroParseContext } from './types.ts'; export function convertLaunchApp( value: unknown, @@ -70,16 +31,20 @@ export function convertLaunchApp( 'permissions', 'launchArguments', ]); - rejectTruthyLaunchOption(value, 'clearState'); - rejectTruthyLaunchOption(value, 'clearKeychain'); - rejectUnsupportedLaunchOption(value, 'arguments'); rejectUnsupportedLaunchOption(value, 'permissions'); - rejectUnsupportedLaunchOption(value, 'launchArguments'); + rejectUnsupportedLaunchOption(value, 'clearKeychain'); const appId = resolveMaestroString( typeof value.appId === 'string' ? value.appId : requireAppId(config, 'launchApp'), context, ); - return action('open', [appId], { relaunch: value.stopApp === true }); + const launchArgs = readLaunchArgs(value, context); + const shouldClearState = value.clearState === true; + const shouldRelaunch = !shouldClearState && (value.stopApp === true || launchArgs.length > 0); + return action('open', [appId], { + ...(shouldRelaunch ? { relaunch: true } : {}), + ...(shouldClearState ? { clearAppState: true } : {}), + ...(launchArgs.length > 0 ? { launchArgs } : {}), + }); } export function convertStopApp( @@ -94,173 +59,32 @@ export function convertStopApp( throw new AppError('INVALID_ARGS', 'stopApp expects a string appId or no value.'); } -export function convertSetAirplaneMode( - value: unknown, - context: MaestroParseContext, -): SessionAction { - const enabled = readBooleanLiteral(resolveMaybeMaestroString(value, context), 'setAirplaneMode'); - return action('settings', ['airplane', enabled ? 'on' : 'off']); -} - -export function convertSetLocation(value: unknown, context: MaestroParseContext): SessionAction { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setLocation expects a map.'); - } - assertOnlyKeys(value, 'setLocation', ['latitude', 'longitude', 'lat', 'lon', 'lng']); - const latitude = readCoordinate(value.latitude ?? value.lat, 'setLocation.latitude', context); - const longitude = readCoordinate( - value.longitude ?? value.lon ?? value.lng, - 'setLocation.longitude', - context, - ); - return action('settings', ['location', 'set', latitude, longitude]); -} - -export function convertSetOrientation(value: unknown, context: MaestroParseContext): SessionAction { - const raw = resolveMaybeMaestroString(value, context); - if (typeof raw !== 'string') { - throw new AppError('INVALID_ARGS', 'setOrientation expects a string value.'); - } - const orientation = normalizeToken(raw); - switch (orientation) { - case 'portrait': - case 'landscape-left': - case 'landscape-right': - return action('rotate', [orientation]); - case 'portrait-upside-down': - case 'upside-down': - return action('rotate', ['portrait-upside-down']); - default: - throw unsupportedMaestroSyntax( - `Maestro setOrientation "${raw}" cannot be mapped to a supported rotate orientation.`, - ); - } +function readLaunchArgs(value: Record, context: MaestroParseContext): string[] { + return [ + ...readLaunchArgValue(value.arguments, 'launchApp.arguments', context), + ...readLaunchArgValue(value.launchArguments, 'launchApp.launchArguments', context), + ]; } -export function convertSetPermissions( - value: unknown, - context: MaestroParseContext, -): SessionAction[] { - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'setPermissions expects a map.'); +function readLaunchArgValue(value: unknown, name: string, context: MaestroParseContext): string[] { + if (value === undefined || value === null) return []; + if (typeof value === 'string') return [resolveMaestroString(value, context)]; + if (Array.isArray(value)) { + return value.map((entry, index) => readLaunchArgScalar(entry, `${name}[${index}]`, context)); } - return Object.entries(value).map(([rawTarget, rawState]) => { - const { target, command, mode } = readPermissionMapping(rawTarget, rawState, context); - return action('settings', ['permission', command, target, ...(mode ? [mode] : [])]); - }); -} - -export function convertKillApp( - value: unknown, - config: MaestroFlowConfig, - context: MaestroParseContext, -): SessionAction { - if (value === null || value === undefined) { - return action('close', [resolveMaestroString(requireAppId(config, 'killApp'), context)]); + if (isPlainRecord(value)) { + return Object.entries(value).flatMap(([key, entry]) => [ + resolveMaestroString(key, context), + readLaunchArgScalar(entry, `${name}.${key}`, context), + ]); } - if (typeof value === 'string') return action('close', [resolveMaestroString(value, context)]); - throw new AppError('INVALID_ARGS', 'killApp expects a string appId or no value.'); + throw new AppError('INVALID_ARGS', `${name} expects a string, list, or map.`); } -export function convertStartRecording(value: unknown, context: MaestroParseContext): SessionAction { - if (value === null || value === undefined) return action('record', ['start']); - if (typeof value === 'string') - return action('record', ['start', resolveMaestroString(value, context)]); - if (!isPlainRecord(value)) { - throw new AppError('INVALID_ARGS', 'startRecording expects a string path, map, or no value.'); - } - assertOnlyKeys(value, 'startRecording', ['path', 'file']); - const rawPath = value.path ?? value.file; - if (rawPath === undefined) return action('record', ['start']); - if (typeof rawPath !== 'string') { - throw new AppError('INVALID_ARGS', 'startRecording path must be a string.'); - } - return action('record', ['start', resolveMaestroString(rawPath, context)]); -} - -export function convertStopRecording(value: unknown): SessionAction { - if (value !== null && value !== undefined) { - throw new AppError('INVALID_ARGS', 'stopRecording expects no value.'); - } - return action('record', ['stop']); -} - -export function convertAssertTrue(value: unknown, context: MaestroParseContext): SessionAction[] { - const resolved = resolveMaybeMaestroString(value, context); - if (resolved === true || (typeof resolved === 'string' && normalizeToken(resolved) === 'true')) { - return []; - } - if ( - resolved === false || - (typeof resolved === 'string' && normalizeToken(resolved) === 'false') - ) { - throw new AppError('INVALID_ARGS', 'Maestro assertTrue literal evaluated to false.'); - } - throw unsupportedMaestroSyntax('Only literal Maestro assertTrue true/false is supported.'); -} - -function readCoordinate(value: unknown, name: string, context: MaestroParseContext): string { - const resolved = resolveMaybeMaestroString(value, context); - const numeric = - typeof resolved === 'number' - ? resolved - : typeof resolved === 'string' && resolved.trim().length > 0 - ? Number(resolved) - : Number.NaN; - if (!Number.isFinite(numeric)) { - throw new AppError('INVALID_ARGS', `${name} must be a finite number.`); - } - return String(numeric); -} - -function readPermissionMapping( - rawTarget: string, - rawState: unknown, - context: MaestroParseContext, -): { target: string; command: PermissionCommand; mode?: string } { - let target = normalizeToken(rawTarget); - const resolvedState = resolveMaybeMaestroString(rawState, context); - if (typeof resolvedState !== 'string') { - throw new AppError('INVALID_ARGS', `setPermissions.${rawTarget} expects a string state.`); - } - const state = normalizeToken(resolvedState); - if (target === 'location' && state === 'always') target = 'location-always'; - - if (!SUPPORTED_PERMISSION_TARGETS.has(target)) { - throw unsupportedMaestroSyntax( - `Maestro setPermissions target "${rawTarget}" cannot be mapped to a supported settings permission target.`, - ); - } - - const basicCommand = BASIC_PERMISSION_STATES[state]; - if (basicCommand) return { target, command: basicCommand }; - - const modeMapping = MODE_PERMISSION_STATES[state]; - if (modeMapping) return { target, ...modeMapping }; - - const locationCommand = readLocationPermissionCommand(target, state); - if (locationCommand) return { target, command: locationCommand }; - - throw unsupportedMaestroSyntax( - `Maestro setPermissions state "${resolvedState}" cannot be mapped to grant, deny, or reset.`, - ); -} - -function readLocationPermissionCommand( - target: string, - state: string, -): PermissionCommand | undefined { - if (target === 'location-always' && state === 'always') return 'grant'; - if (target === 'location' && (state === 'while-in-use' || state === 'when-in-use')) { - return 'grant'; - } - return undefined; -} - -function rejectTruthyLaunchOption(value: Record, key: string): void { - if (value[key] === true) { - throw unsupportedMaestroSyntax(`Maestro launchApp ${key}: true is not supported yet.`); - } +function readLaunchArgScalar(value: unknown, name: string, context: MaestroParseContext): string { + if (typeof value === 'string') return resolveMaestroString(value, context); + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + throw new AppError('INVALID_ARGS', `${name} must be a string, number, or boolean.`); } function rejectUnsupportedLaunchOption(value: Record, key: string): void { diff --git a/src/compat/maestro/flow-control.ts b/src/compat/maestro/flow-control.ts new file mode 100644 index 000000000..0c526cb93 --- /dev/null +++ b/src/compat/maestro/flow-control.ts @@ -0,0 +1,471 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import type { SessionAction } from '../../daemon/types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { maestroSelector } from './interactions.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + action, + assertOnlyKeys, + isPlainRecord, + normalizeCommandList, + readEnvMap, + resolveMaestroString, + unsupportedMaestroSyntax, +} from './support.ts'; +import type { + MaestroCommand, + MaestroCommandMapperDeps, + MaestroFlowConfig, + MaestroParseContext, +} from './types.ts'; + +// repeat.times is expanded at parse time for deterministic replay traces. Keep +// a guardrail until repeat can execute as a runtime loop without materializing +// every child action. +const MAX_REPEAT_EXPANSIONS = 1000; +type MaestroConditionPlatform = 'android' | 'ios' | 'web'; + +type ConvertCommandList = ( + commands: MaestroCommand[], + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, +) => SessionAction[]; + +export function convertRunFlow( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value, context), context).actions; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow expects a file path string or map.'); + } + assertOnlyKeys(value, 'runFlow', ['file', 'commands', 'env', 'when', 'label']); + const condition = readRunFlowCondition(value.when, context); + if (!condition.shouldRun) return []; + + const runContext = { + ...context, + env: { ...context.env, ...readEnvMap(value.env, 'runFlow.env'), ...context.envOverrides }, + }; + const actions = readRunFlowActions(value, config, runContext, deps, convertCommandList); + return wrapRunFlowCondition(actions, condition); +} + +export function convertRepeat( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'repeat expects a map.'); + } + assertOnlyKeys(value, 'repeat', ['times', 'commands', 'while']); + if (value.while !== undefined) { + throw unsupportedMaestroSyntax( + 'Maestro repeat.while is not supported yet. Only deterministic repeat.times is supported.', + ); + } + const times = readRepeatTimes(value.times, context); + if (!Array.isArray(value.commands)) { + throw new AppError('INVALID_ARGS', 'repeat requires a commands list.'); + } + if (times > MAX_REPEAT_EXPANSIONS) { + throw new AppError( + 'INVALID_ARGS', + `repeat.times must be <= ${MAX_REPEAT_EXPANSIONS} for deterministic replay expansion.`, + ); + } + const commands = normalizeCommandList(value.commands); + return Array.from({ length: times }).flatMap(() => + convertCommandList(commands, config, context, deps), + ); +} + +export function convertRetry( + value: unknown, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'retry expects a map.'); + } + assertOnlyKeys(value, 'retry', ['maxRetries', 'commands']); + if (!Array.isArray(value.commands)) { + throw new AppError('INVALID_ARGS', 'retry requires a commands list.'); + } + const maxRetries = readRetryMaxRetries(value.maxRetries, context); + const commands = normalizeCommandList(value.commands); + const actions = convertCommandList(commands, config, context, deps); + return [ + action(MAESTRO_RUNTIME_COMMAND.retry, [String(maxRetries)], { + batchSteps: actions.map(sessionActionToBatchStep), + }), + ]; +} + +function readRunFlowActions( + value: Record, + config: MaestroFlowConfig, + context: MaestroParseContext, + deps: MaestroCommandMapperDeps, + convertCommandList: ConvertCommandList, +): SessionAction[] { + if (typeof value.file === 'string') { + return deps.parseRunFlowFile(resolveMaestroString(value.file, context), context).actions; + } + if (Array.isArray(value.commands)) { + return convertCommandList(normalizeCommandList(value.commands), config, context, deps); + } + throw new AppError('INVALID_ARGS', 'runFlow map requires either file or commands.'); +} + +type RunFlowCondition = { + shouldRun: boolean; + visibleSelector?: string; + notVisibleSelector?: string; +}; + +function readRunFlowCondition(value: unknown, context: MaestroParseContext): RunFlowCondition { + if (value === undefined || value === null) return { shouldRun: true }; + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runFlow.when expects a map.'); + } + assertOnlyKeys(value, 'runFlow.when', ['platform', 'visible', 'notVisible', 'true']); + if (!matchesRunFlowStaticCondition(value, context)) return { shouldRun: false }; + return { + shouldRun: true, + ...readRunFlowVisibilityCondition(value, context), + }; +} + +function matchesRunFlowStaticCondition( + value: Record, + context: MaestroParseContext, +): boolean { + if (value.true !== undefined && !evaluateRunFlowTrueCondition(value.true, context)) return false; + if (value.platform === undefined) return true; + const platform = normalizeRunFlowPlatform(value.platform, 'runFlow.when.platform'); + if (!context.platform) { + throw new AppError( + 'INVALID_ARGS', + 'Maestro runFlow.when.platform requires replay to be run with --platform ios|android.', + ); + } + return platform === context.platform; +} + +function readRunFlowVisibilityCondition( + value: Record, + context: MaestroParseContext, +): Pick { + return { + ...(value.visible !== undefined + ? { visibleSelector: maestroSelector(value.visible, 'runFlow.when.visible', [], context) } + : {}), + ...(value.notVisible !== undefined + ? { + notVisibleSelector: maestroSelector( + value.notVisible, + 'runFlow.when.notVisible', + [], + context, + ), + } + : {}), + }; +} + +function normalizeRunFlowPlatform(value: unknown, name: string): MaestroConditionPlatform { + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', `${name} expects Android, iOS, or Web.`); + } + const normalized = value.trim().toLowerCase(); + if (normalized === 'android' || normalized === 'ios' || normalized === 'web') { + return normalized; + } + throw new AppError('INVALID_ARGS', `${name} expects Android, iOS, or Web.`); +} + +function evaluateRunFlowTrueCondition(value: unknown, context: MaestroParseContext): boolean { + if (typeof value === 'boolean') return value; + if (typeof value !== 'string') { + throw new AppError('INVALID_ARGS', 'runFlow.when.true expects a boolean or expression string.'); + } + const expression = unwrapMaestroExpression(resolveMaestroString(value, context)); + const parser = new MaestroBooleanExpressionParser(tokenizeMaestroBooleanExpression(expression), { + platform: context.platform, + }); + return parser.parse(); +} + +function unwrapMaestroExpression(value: string): string { + const trimmed = value.trim(); + return trimmed.startsWith('${') && trimmed.endsWith('}') ? trimmed.slice(2, -1).trim() : trimmed; +} + +type MaestroBooleanToken = + | { type: 'platform' } + | MaestroBooleanOperatorToken + | { type: 'paren'; value: '(' | ')' } + | { type: 'string'; value: string } + | { type: 'boolean'; value: boolean }; + +type MaestroBooleanOperatorToken = { type: 'operator'; value: '==' | '!=' | '&&' | '||' }; + +type MaestroBooleanTokenMatch = { + token: MaestroBooleanToken; + length: number; +}; + +function tokenizeMaestroBooleanExpression(expression: string): MaestroBooleanToken[] { + const tokens: MaestroBooleanToken[] = []; + let index = 0; + while (index < expression.length) { + const remaining = expression.slice(index); + const skipped = whitespaceLength(remaining); + if (skipped > 0) { + index += skipped; + continue; + } + const next = readMaestroBooleanToken(remaining); + if (next) { + tokens.push(next.token); + index += next.length; + continue; + } + throw new AppError( + 'INVALID_ARGS', + `Unsupported runFlow.when.true expression near "${remaining.slice(0, 24)}".`, + ); + } + return tokens; +} + +function whitespaceLength(value: string): number { + return /^\s+/.exec(value)?.[0].length ?? 0; +} + +function readMaestroBooleanToken(remaining: string): MaestroBooleanTokenMatch | null { + return ( + readPlatformToken(remaining) ?? + readOperatorToken(remaining) ?? + readParenToken(remaining) ?? + readStringToken(remaining) ?? + readBooleanToken(remaining) + ); +} + +function readPlatformToken(remaining: string): MaestroBooleanTokenMatch | null { + const name = 'maestro.platform'; + return remaining.startsWith(name) ? { token: { type: 'platform' }, length: name.length } : null; +} + +function readOperatorToken(remaining: string): MaestroBooleanTokenMatch | null { + const operator = /^(==|!=|&&|\|\|)/.exec(remaining)?.[1]; + return operator + ? { + token: { type: 'operator', value: operator as MaestroBooleanOperatorToken['value'] }, + length: operator.length, + } + : null; +} + +function readParenToken(remaining: string): MaestroBooleanTokenMatch | null { + const value = remaining[0]; + return value === '(' || value === ')' ? { token: { type: 'paren', value }, length: 1 } : null; +} + +function readStringToken(remaining: string): MaestroBooleanTokenMatch | null { + const quoted = /^(['"])(.*?)\1/.exec(remaining); + return quoted + ? { token: { type: 'string', value: quoted[2] ?? '' }, length: quoted[0].length } + : null; +} + +function readBooleanToken(remaining: string): MaestroBooleanTokenMatch | null { + const value = /^(true|false)\b/.exec(remaining)?.[1]; + return value + ? { token: { type: 'boolean', value: value === 'true' }, length: value.length } + : null; +} + +class MaestroBooleanExpressionParser { + private index = 0; + private readonly tokens: MaestroBooleanToken[]; + private readonly context: { platform?: 'android' | 'ios' }; + + constructor(tokens: MaestroBooleanToken[], context: { platform?: 'android' | 'ios' }) { + this.tokens = tokens; + this.context = context; + } + + parse(): boolean { + const result = this.parseOr(); + if (this.peek()) { + throw new AppError('INVALID_ARGS', 'Unsupported trailing runFlow.when.true expression.'); + } + return result; + } + + private parseOr(): boolean { + let result = this.parseAnd(); + while (this.consumeOperator('||')) { + result = this.parseAnd() || result; + } + return result; + } + + private parseAnd(): boolean { + let result = this.parsePrimary(); + while (this.consumeOperator('&&')) { + result = this.parsePrimary() && result; + } + return result; + } + + private parsePrimary(): boolean { + const token = this.peek(); + if (!token) { + throw new AppError('INVALID_ARGS', 'Incomplete runFlow.when.true expression.'); + } + if (token.type === 'boolean') { + this.index += 1; + return token.value; + } + if (token.type === 'paren' && token.value === '(') { + this.index += 1; + const result = this.parseOr(); + if (!this.consumeParen(')')) { + throw new AppError('INVALID_ARGS', 'Unclosed runFlow.when.true parenthesis.'); + } + return result; + } + return this.parsePlatformComparison(); + } + + private parsePlatformComparison(): boolean { + this.expectPlatform(); + const operator = this.expectEqualityOperator(); + const value = this.expectString().toLowerCase(); + const platform = this.context.platform; + return operator === '==' ? platform === value : platform !== value; + } + + private expectPlatform(): void { + if (this.peek()?.type !== 'platform') { + throw new AppError( + 'INVALID_ARGS', + 'runFlow.when.true supports maestro.platform comparisons.', + ); + } + this.index += 1; + } + + private expectEqualityOperator(): '==' | '!=' { + const token = this.peek(); + if (token?.type === 'operator' && (token.value === '==' || token.value === '!=')) { + this.index += 1; + return token.value; + } + throw new AppError('INVALID_ARGS', 'runFlow.when.true comparison requires == or !=.'); + } + + private expectString(): string { + const token = this.peek(); + if (token?.type === 'string') { + this.index += 1; + return token.value; + } + throw new AppError('INVALID_ARGS', 'runFlow.when.true comparison requires a string literal.'); + } + + private consumeOperator(value: '&&' | '||'): boolean { + const token = this.peek(); + if (token?.type !== 'operator' || token.value !== value) return false; + this.index += 1; + return true; + } + + private consumeParen(value: '(' | ')'): boolean { + const token = this.peek(); + if (token?.type !== 'paren' || token.value !== value) return false; + this.index += 1; + return true; + } + + private peek(): MaestroBooleanToken | undefined { + return this.tokens[this.index]; + } +} + +function wrapRunFlowCondition( + actions: SessionAction[], + condition: RunFlowCondition, +): SessionAction[] { + if (!condition.visibleSelector && !condition.notVisibleSelector) return actions; + if (condition.visibleSelector && condition.notVisibleSelector) { + throw unsupportedMaestroSyntax( + 'Maestro runFlow.when cannot combine visible and notVisible yet.', + ); + } + return [ + action( + MAESTRO_RUNTIME_COMMAND.runFlowWhen, + condition.visibleSelector + ? ['visible', condition.visibleSelector] + : ['notVisible', condition.notVisibleSelector ?? ''], + { batchSteps: actions.map(sessionActionToBatchStep) }, + ), + ]; +} + +function sessionActionToBatchStep( + entry: SessionAction, +): NonNullable[number] { + return { + command: entry.command, + positionals: entry.positionals, + flags: entry.flags, + ...(entry.runtime !== undefined ? { runtime: entry.runtime } : {}), + }; +} + +function readRepeatTimes(value: unknown, context: MaestroParseContext): number { + return readMaestroNonNegativeInteger(value, context, 'repeat.times'); +} + +function readRetryMaxRetries(value: unknown, context: MaestroParseContext): number { + if (value === undefined) return 1; + return readMaestroNonNegativeInteger(value, context, 'retry.maxRetries'); +} + +function readMaestroNonNegativeInteger( + value: unknown, + context: MaestroParseContext, + name: string, +): number { + const resolved = typeof value === 'string' ? resolveMaestroString(value, context) : value; + const numeric = + typeof resolved === 'number' + ? resolved + : typeof resolved === 'string' && /^\d+$/.test(resolved) + ? Number(resolved) + : undefined; + if (numeric === undefined || !Number.isInteger(numeric) || numeric < 0) { + throw new AppError( + 'INVALID_ARGS', + `${name} must be a non-negative integer or \${VAR} resolving to one.`, + ); + } + return numeric; +} diff --git a/src/compat/maestro/interactions.ts b/src/compat/maestro/interactions.ts index c61271606..6448d2c26 100644 --- a/src/compat/maestro/interactions.ts +++ b/src/compat/maestro/interactions.ts @@ -9,19 +9,39 @@ import { resolveMaestroString, unsupportedMaestroSyntax, } from './support.ts'; +import { parseAbsolutePoint, parseMaestroPoint } from './points.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import type { MaestroParseContext } from './types.ts'; +type SwipeDirection = 'up' | 'down' | 'left' | 'right'; + export function convertTapOn(value: unknown, context: MaestroParseContext): SessionAction { + if (typeof value === 'string') { + return action( + MAESTRO_RUNTIME_COMMAND.tapOn, + [visibleTextSelector(resolveMaestroString(value, context))], + maestroTapOnFlags(value), + ); + } if (isPlainRecord(value) && typeof value.point === 'string') { - assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay']); - const point = parsePoint(value.point); + assertOnlyKeys(value, 'tapOn', ['point', 'repeat', 'delay', 'optional', 'label']); + const point = parseMaestroPoint(value.point); + if (point.kind === 'percent') { + return action( + MAESTRO_RUNTIME_COMMAND.tapPointPercent, + [String(point.x), String(point.y)], + tapFlags(value), + ); + } return action('click', [String(point.x), String(point.y)], tapFlags(value)); } if (isPlainRecord(value)) { assertOnlyKeys(value, 'tapOn', [ 'id', 'text', + 'childOf', 'enabled', + 'index', 'selected', 'repeat', 'delay', @@ -29,17 +49,26 @@ export function convertTapOn(value: unknown, context: MaestroParseContext): Sess 'label', ]); } + const flags = maestroTapOnFlags(value); return action( - 'click', - [maestroSelector(value, 'tapOn', ['repeat', 'delay', 'optional', 'label'], context)], - tapFlags(value), + MAESTRO_RUNTIME_COMMAND.tapOn, + [ + maestroSelector( + value, + 'tapOn', + ['repeat', 'delay', 'optional', 'label', 'index', 'childOf'], + context, + ), + ...maestroTapOnRuntimeOptions(value, context), + ], + flags, ); } export function convertDoubleTapOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'doubleTapOn', ['point', 'delay']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('click', [String(point.x), String(point.y)], doubleTapFlags(value)); } if (isPlainRecord(value)) { @@ -55,7 +84,7 @@ export function convertDoubleTapOn(value: unknown, context: MaestroParseContext) export function convertLongPressOn(value: unknown, context: MaestroParseContext): SessionAction { if (isPlainRecord(value) && typeof value.point === 'string') { assertOnlyKeys(value, 'longPressOn', ['point']); - const point = parsePoint(value.point); + const point = parseAbsolutePoint(value.point); return action('longpress', [String(point.x), String(point.y), '3000']); } if (isPlainRecord(value)) { @@ -76,6 +105,26 @@ export function readInputText(value: unknown): string { return value.text; } +export function convertEraseText(value: unknown): SessionAction { + if (value === null || value === undefined) return action('type', ['\b'.repeat(50)]); + if (typeof value === 'number' && Number.isInteger(value) && value > 0) { + return action('type', ['\b'.repeat(value)]); + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'eraseText expects empty, a positive count, or a map.'); + } + assertOnlyKeys(value, 'eraseText', ['charactersToErase']); + if (value.charactersToErase === undefined) return action('type', ['\b'.repeat(50)]); + if ( + typeof value.charactersToErase !== 'number' || + !Number.isInteger(value.charactersToErase) || + value.charactersToErase <= 0 + ) { + throw new AppError('INVALID_ARGS', 'eraseText.charactersToErase must be a positive integer.'); + } + return action('type', ['\b'.repeat(value.charactersToErase)]); +} + export function convertExtendedWaitUntil( value: unknown, context: MaestroParseContext, @@ -95,7 +144,7 @@ export function convertExtendedWaitUntil( if (value.notVisible !== undefined) { return [action('wait', [timeoutMs]), action('is', ['hidden', selector])]; } - return [action('wait', [selector, timeoutMs])]; + return [action(MAESTRO_RUNTIME_COMMAND.assertVisible, [selector, timeoutMs])]; } export function convertScroll(value: unknown): SessionAction { @@ -105,20 +154,93 @@ export function convertScroll(value: unknown): SessionAction { return action('scroll', ['down']); } -export function convertSwipe(value: unknown): SessionAction { +export function convertScrollUntilVisible( + value: unknown, + context: MaestroParseContext, +): SessionAction[] { + if (typeof value === 'string') { + return [ + action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [ + visibleTextSelector(resolveMaestroString(value, context)), + '5000', + 'down', + ]), + ]; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'scrollUntilVisible expects a string or map.'); + } + assertOnlyKeys(value, 'scrollUntilVisible', ['element', 'direction', 'timeout']); + const selector = maestroSelector(value.element, 'scrollUntilVisible.element', [], context); + const direction = + typeof value.direction === 'string' + ? readMaestroDirection(value.direction, 'scrollUntilVisible.direction') + : 'down'; + const timeoutMs = String(readTimeoutMs(value, 5000)); + return [action(MAESTRO_RUNTIME_COMMAND.scrollUntilVisible, [selector, timeoutMs, direction])]; +} + +export function convertSwipe(value: unknown, context: MaestroParseContext): SessionAction { if (!isPlainRecord(value)) { throw new AppError('INVALID_ARGS', 'swipe expects a map.'); } - assertOnlyKeys(value, 'swipe', ['start', 'end', 'duration']); - if (typeof value.start !== 'string' || typeof value.end !== 'string') { - throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); + assertOnlyKeys(value, 'swipe', ['start', 'end', 'direction', 'duration', 'from', 'label']); + const from = value.from ?? (typeof value.label === 'string' ? value.label : undefined); + if (from !== undefined) { + return convertTargetedSwipe(value, from, context); } - const start = parseSwipePoint(value.start); - const end = parseSwipePoint(value.end); - const durationMs = - typeof value.duration === 'number' && Number.isFinite(value.duration) - ? String(Math.max(16, Math.floor(value.duration))) - : undefined; + if (typeof value.direction === 'string') { + return action(MAESTRO_RUNTIME_COMMAND.swipeScreen, [ + 'direction', + readSwipeDirection(value.direction), + ...swipeDurationPositionals(value), + ]); + } + return convertCoordinateSwipe(value); +} + +function convertTargetedSwipe( + value: Record, + from: unknown, + context: MaestroParseContext, +): SessionAction { + const direction = readSwipeDirection( + typeof value.direction === 'string' ? value.direction : 'up', + ); + return action(MAESTRO_RUNTIME_COMMAND.swipeOn, [ + maestroSelector(from, 'swipe.from', [], context), + direction, + ...swipeDurationPositionals(value), + ]); +} + +function convertCoordinateSwipe(value: Record): SessionAction { + const { start, end } = readCoordinateSwipePoints(value); + const durationMs = readSwipeDurationMs(value.duration); + return convertCoordinateSwipePoints(start, end, durationMs); +} + +function readCoordinateSwipePoints(value: Record): { + start: ReturnType; + end: ReturnType; +} { + if (typeof value.start === 'string' && typeof value.end === 'string') { + return { start: parseMaestroPoint(value.start), end: parseMaestroPoint(value.end) }; + } + throw unsupportedMaestroSyntax('Only Maestro swipe start/end coordinates are supported.'); +} + +function readSwipeDurationMs(duration: unknown): string | undefined { + return typeof duration === 'number' && Number.isFinite(duration) + ? String(Math.max(16, Math.floor(duration))) + : undefined; +} + +function convertCoordinateSwipePoints( + start: ReturnType, + end: ReturnType, + durationMs: string | undefined, +): SessionAction { if (start.kind === 'absolute' && end.kind === 'absolute') { return action('swipe', [ String(start.x), @@ -129,17 +251,41 @@ export function convertSwipe(value: unknown): SessionAction { ]); } if (start.kind === 'percent' && end.kind === 'percent') { - return action('scroll', readScrollPositionalsFromPercentSwipe(start, end)); + return action(MAESTRO_RUNTIME_COMMAND.swipeScreen, [ + 'percent', + String(start.x), + String(start.y), + String(end.x), + String(end.y), + ...(durationMs ? [durationMs] : []), + ]); } throw unsupportedMaestroSyntax( 'Maestro swipe start/end must both be absolute pixels or both be percentages.', ); } +function readMaestroDirection(direction: string, name: string): SwipeDirection { + const normalized = direction.toLowerCase(); + switch (normalized) { + case 'up': + case 'down': + case 'left': + case 'right': + return normalized; + default: + throw unsupportedMaestroSyntax(`Maestro ${name} must be UP, DOWN, LEFT, or RIGHT.`); + } +} + +function readSwipeDirection(direction: string): SwipeDirection { + return readMaestroDirection(direction, 'swipe direction'); +} + export function convertPressKey(value: unknown): SessionAction { const key = requireStringValue('pressKey', value).toLowerCase(); if (key === 'back') return action('back'); - if (key === 'enter' || key === 'return') return action('press', ['return']); + if (key === 'enter' || key === 'return') return action(MAESTRO_RUNTIME_COMMAND.pressEnter); if (key === 'home') return action('home'); throw unsupportedMaestroSyntax(`Maestro pressKey "${key}" is not supported yet.`); } @@ -157,31 +303,56 @@ export function maestroSelector( assertOnlyKeys(value, command, ['id', 'text', 'enabled', 'selected', ...allowedExtraKeys]); const terms: string[] = []; - if (typeof value.id === 'string') - terms.push(selectorTerm('id', resolveMaestroString(value.id, context))); - if (typeof value.text === 'string') - terms.push(selectorTerm('label', resolveMaestroString(value.text, context))); + const stateTerms: string[] = []; if (typeof value.enabled === 'boolean') - terms.push(selectorTerm('enabled', String(value.enabled))); + stateTerms.push(selectorTerm('enabled', String(value.enabled))); if (typeof value.selected === 'boolean') - terms.push(selectorTerm('selected', String(value.selected))); + stateTerms.push(selectorTerm('selected', String(value.selected))); + if (typeof value.id === 'string') + terms.push(selectorTerm('id', resolveMaestroString(value.id, context)), ...stateTerms); + if (typeof value.text === 'string' && terms.length === 0) { + return visibleTextSelector(resolveMaestroString(value.text, context), stateTerms); + } + if (typeof value.label === 'string' && terms.length === 0) + terms.push(selectorTerm('label', resolveMaestroString(value.label, context)), ...stateTerms); + if (terms.length === 0 && stateTerms.length > 0) terms.push(...stateTerms); if (terms.length === 0) { throw new AppError( 'INVALID_ARGS', - `${command} selector map must include one of id, text, enabled, or selected.`, + `${command} selector map must include one of id, text, label, enabled, or selected.`, ); } return terms.join(' '); } -function visibleTextSelector(value: string): string { +function visibleTextSelector(value: string, extraTerms: readonly string[] = []): string { return [ - selectorTerm('label', value), - selectorTerm('text', value), - selectorTerm('id', value), + [selectorTerm('label', value), ...extraTerms].join(' '), + [selectorTerm('text', value), ...extraTerms].join(' '), + [selectorTerm('id', value), ...extraTerms].join(' '), ].join(' || '); } +function maestroTapOnRuntimeOptions(value: unknown, context: MaestroParseContext): string[] { + if (!isPlainRecord(value)) return []; + const options: { index?: number; childOf?: string } = {}; + if (value.index !== undefined) { + if (typeof value.index !== 'number' || !Number.isInteger(value.index) || value.index < 0) { + throw new AppError('INVALID_ARGS', 'tapOn.index must be a non-negative integer.'); + } + options.index = value.index; + } + if (value.childOf !== undefined) { + options.childOf = maestroSelector(value.childOf, 'tapOn.childOf', [], context); + } + return Object.keys(options).length > 0 ? [JSON.stringify(options)] : []; +} + +function swipeDurationPositionals(value: Record): string[] { + const durationMs = readSwipeDurationMs(value.duration); + return durationMs ? [durationMs] : []; +} + function selectorTerm(key: string, value: string): string { return `${key}=${JSON.stringify(value)}`; } @@ -189,15 +360,25 @@ function selectorTerm(key: string, value: string): string { function tapFlags(value: unknown): SessionAction['flags'] | undefined { if (!isPlainRecord(value)) return undefined; const flags: SessionAction['flags'] = {}; - if (typeof value.repeat === 'number' && Number.isInteger(value.repeat) && value.repeat > 1) { - flags.count = value.repeat; - } - if (typeof value.delay === 'number' && Number.isInteger(value.delay) && value.delay >= 0) { - flags.intervalMs = value.delay; - } + const repeat = positiveInteger(value.repeat); + const delay = nonNegativeInteger(value.delay); + if (repeat && repeat > 1) flags.count = repeat; + if (delay !== undefined) flags.intervalMs = delay; + if (value.optional === true) flags.maestro = { optional: true }; return Object.keys(flags).length > 0 ? flags : undefined; } +function maestroTapOnFlags(value: unknown): SessionAction['flags'] { + const flags = tapFlags(value) ?? {}; + return { + ...flags, + maestro: { + ...(flags.maestro ?? {}), + allowNonHittableCoordinateFallback: true, + }, + }; +} + function doubleTapFlags(value: unknown): SessionAction['flags'] { const flags: SessionAction['flags'] = { doubleTap: true }; if (isPlainRecord(value) && typeof value.delay === 'number' && Number.isInteger(value.delay)) { @@ -206,57 +387,10 @@ function doubleTapFlags(value: unknown): SessionAction['flags'] { return flags; } -function parsePoint(value: string): { x: number; y: number } { - const match = value.match(/^(\d+),(\d+)$/); - if (!match) { - throw unsupportedMaestroSyntax( - 'Only absolute Maestro point selectors like "100,200" are supported.', - ); - } - return { x: Number(match[1]), y: Number(match[2]) }; -} - -type SwipePoint = - | { - kind: 'absolute'; - x: number; - y: number; - } - | { - kind: 'percent'; - x: number; - y: number; - }; - -function parseSwipePoint(value: string): SwipePoint { - const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); - if (absolute) { - return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; - } - const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); - if (percent) { - return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; - } - throw unsupportedMaestroSyntax( - 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', - ); -} - -function readScrollPositionalsFromPercentSwipe( - start: Extract, - end: Extract, -): string[] { - const deltaX = end.x - start.x; - const deltaY = end.y - start.y; - if (Math.abs(deltaX) === 0 && Math.abs(deltaY) === 0) { - throw new AppError('INVALID_ARGS', 'swipe start and end cannot be the same point.'); - } - const vertical = Math.abs(deltaY) >= Math.abs(deltaX); - const direction = vertical ? (deltaY < 0 ? 'down' : 'up') : deltaX < 0 ? 'right' : 'left'; - const amount = Math.min(1, Math.max(0.01, Math.abs(vertical ? deltaY : deltaX) / 100)); - return [direction, formatAmount(amount)]; +function positiveInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; } -function formatAmount(value: number): string { - return value.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); +function nonNegativeInteger(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value >= 0 ? value : undefined; } diff --git a/src/compat/maestro/points.ts b/src/compat/maestro/points.ts new file mode 100644 index 000000000..571e692a5 --- /dev/null +++ b/src/compat/maestro/points.ts @@ -0,0 +1,37 @@ +import { unsupportedMaestroSyntax } from './support.ts'; + +export type MaestroPoint = + | { + kind: 'absolute'; + x: number; + y: number; + } + | { + kind: 'percent'; + x: number; + y: number; + }; + +export function parseAbsolutePoint(value: string): { x: number; y: number } { + const match = value.match(/^(\d+),(\d+)$/); + if (!match) { + throw unsupportedMaestroSyntax( + 'Only absolute Maestro point selectors like "100,200" are supported.', + ); + } + return { x: Number(match[1]), y: Number(match[2]) }; +} + +export function parseMaestroPoint(value: string): MaestroPoint { + const absolute = value.match(/^\s*(\d+)\s*,\s*(\d+)\s*$/); + if (absolute) { + return { kind: 'absolute', x: Number(absolute[1]), y: Number(absolute[2]) }; + } + const percent = value.match(/^\s*(\d+(?:\.\d+)?)%\s*,\s*(\d+(?:\.\d+)?)%\s*$/); + if (percent) { + return { kind: 'percent', x: Number(percent[1]), y: Number(percent[2]) }; + } + throw unsupportedMaestroSyntax( + 'Only Maestro swipe coordinates like "100,200" or "50%,75%" are supported.', + ); +} diff --git a/src/compat/maestro/replay-flow.ts b/src/compat/maestro/replay-flow.ts index e6b28cdaa..cc247d22c 100644 --- a/src/compat/maestro/replay-flow.ts +++ b/src/compat/maestro/replay-flow.ts @@ -4,6 +4,7 @@ import { parseAllDocuments } from 'yaml'; import type { SessionAction } from '../../daemon/types.ts'; import { AppError } from '../../utils/errors.ts'; import { convertMaestroCommandWithLine } from './command-mapper.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; import { isPlainRecord, normalizeCommandList, normalizePlatform, readEnvMap } from './support.ts'; import type { MaestroCommand, @@ -74,7 +75,100 @@ function convertRootCommands(params: { actions.push(...converted); converted.forEach(() => actionLines.push(line)); } - return { actions, actionLines }; + return optimizeInputTextActions(actions, actionLines); +} + +function optimizeInputTextActions( + actions: SessionAction[], + actionLines: number[], +): { actions: SessionAction[]; actionLines: number[] } { + const mergedActions: SessionAction[] = []; + const mergedLines: number[] = []; + for (let index = 0; index < actions.length; index += 1) { + const action = actions[index]; + const optimized = optimizeTypedAfterTap(actions, actionLines, index); + if (optimized) { + mergedActions.push(...optimized.actions); + mergedLines.push(...optimized.actionLines); + index += optimized.consumed - 1; + continue; + } + mergedActions.push(action); + mergedLines.push(actionLines[index] ?? 1); + } + return { actions: mergedActions, actionLines: mergedLines }; +} + +function optimizeTypedAfterTap( + actions: SessionAction[], + actionLines: number[], + index: number, +): { actions: SessionAction[]; actionLines: number[]; consumed: number } | null { + const action = actions[index]; + const nextAction = actions[index + 1]; + const typedAfterTap = readPlainTypeText(nextAction); + const tapSelector = readPlainMaestroTapSelector(action); + if (typedAfterTap === null || tapSelector === null) return null; + const line = actionLines[index] ?? 1; + if (!isLikelyTextEntrySelector(tapSelector)) { + return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; + } + if (actions[index + 2]?.command !== MAESTRO_RUNTIME_COMMAND.pressEnter) { + return { actions: [clearMaestroNonHittableTap(action)], actionLines: [line], consumed: 1 }; + } + return { + actions: [ + { + ...action, + command: 'wait', + positionals: [tapSelector, '30000'], + }, + { + ...nextAction, + command: 'fill', + positionals: [tapSelector, typedAfterTap], + flags: action.flags, + }, + actions[index + 2] as SessionAction, + ], + actionLines: [line, line, actionLines[index + 2] ?? line], + consumed: 3, + }; +} + +function clearMaestroNonHittableTap(action: SessionAction): SessionAction { + const maestro = { ...(action.flags?.maestro ?? {}) }; + delete maestro.allowNonHittableCoordinateFallback; + return { + ...action, + flags: { + ...(action.flags ?? {}), + maestro: { + ...maestro, + }, + }, + }; +} + +function readPlainMaestroTapSelector(action: SessionAction | undefined): string | null { + if (action?.command !== MAESTRO_RUNTIME_COMMAND.tapOn) return null; + const [selector, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof selector !== 'string') return null; + return selector; +} + +function readPlainTypeText(action: SessionAction | undefined): string | null { + if (action?.command !== 'type') return null; + if (action.flags && Object.keys(action.flags).length > 0) return null; + const [text, ...rest] = action.positionals ?? []; + if (rest.length > 0 || typeof text !== 'string') return null; + return text; +} + +function isLikelyTextEntrySelector(selector: string): boolean { + return /\b(input|textfield|textarea|field|email|password|username|search|query)\b/i.test( + selector.replace(/([a-z])([A-Z])/g, '$1 $2'), + ); } function parseYamlDocuments(script: string): unknown[] { diff --git a/src/compat/maestro/run-script.ts b/src/compat/maestro/run-script.ts new file mode 100644 index 000000000..2b6602690 --- /dev/null +++ b/src/compat/maestro/run-script.ts @@ -0,0 +1,229 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import vm from 'node:vm'; +import type { SessionAction } from '../../daemon/types.ts'; +import { AppError } from '../../utils/errors.ts'; +import { runCmdSync } from '../../utils/exec.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + action, + assertOnlyKeys, + isPlainRecord, + readEnvMap, + requireStringValue, + resolveMaestroString, +} from './support.ts'; +import type { MaestroParseContext } from './types.ts'; + +const RUN_SCRIPT_TIMEOUT_MS = 30_000; + +type HttpResponse = { + status: number; + body: string; + headers: Record; +}; + +const HTTP_REQUEST_SCRIPT = ` +const fs = require('node:fs'); +const input = JSON.parse(fs.readFileSync(0, 'utf8')); +if (typeof fetch !== 'function') { + console.error('global fetch is required for Maestro runScript http helpers'); + process.exit(1); +} +fetch(input.url, { + method: input.method, + headers: input.headers, + body: input.body, +}).then(async response => { + process.stdout.write(JSON.stringify({ + status: response.status, + body: await response.text(), + headers: Object.fromEntries(response.headers.entries()), + })); +}).catch(error => { + console.error(error && error.stack ? error.stack : String(error)); + process.exit(1); +}); +`; + +export function convertRunScript(value: unknown, context: MaestroParseContext): SessionAction { + const scriptConfig = readRunScriptConfig(value, context); + const scriptPath = resolveRunScriptPath(scriptConfig.file, context); + return action(MAESTRO_RUNTIME_COMMAND.runScript, [scriptPath], { + ...(Object.keys(scriptConfig.env).length > 0 + ? { maestro: { runScriptEnv: scriptConfig.env } } + : {}), + }); +} + +export function executeRunScriptFile(params: { + scriptPath: string; + env: Record; +}): Record { + const { scriptPath, env } = params; + const script = fs.readFileSync(scriptPath, 'utf8'); + const output: Record = Object.create(null) as Record; + + try { + // Compatibility note: node:vm is not a security sandbox. Maestro runScript + // files are trusted flow-local setup code; the timeout only bounds + // synchronous script execution. Async http.post work is bounded separately + // by the child process timeout in runHttpRequestSync. + vm.runInNewContext(script, buildScriptGlobals(env, output), { + filename: scriptPath, + timeout: RUN_SCRIPT_TIMEOUT_MS, + }); + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript failed for ${scriptPath}: ${error instanceof Error ? error.message : String(error)}`, + { scriptPath }, + error instanceof Error ? error : undefined, + ); + } + + validateOutputKeys(output, scriptPath); + return Object.fromEntries( + Object.entries(output).map(([key, rawValue]) => [ + `output.${key}`, + stringifyOutputValue(rawValue), + ]), + ); +} + +function readRunScriptConfig( + value: unknown, + context: MaestroParseContext, +): { file: string; env: Record } { + if (typeof value === 'string') { + return { file: resolveMaestroString(value, context), env: {} }; + } + if (!isPlainRecord(value)) { + throw new AppError('INVALID_ARGS', 'runScript expects a file path string or map.'); + } + assertOnlyKeys(value, 'runScript', ['file', 'env']); + const file = resolveMaestroString(requireStringValue('runScript.file', value.file), context); + const rawEnv = readEnvMap(value.env, 'runScript.env'); + const env = Object.fromEntries( + Object.entries(rawEnv).map(([key, envValue]) => [key, resolveMaestroString(envValue, context)]), + ); + return { file, env }; +} + +function resolveRunScriptPath(filePath: string, context: MaestroParseContext): string { + if (path.isAbsolute(filePath)) return filePath; + if (!context.baseDir) { + throw new AppError( + 'INVALID_ARGS', + 'runScript file paths require replay input to have a source path.', + ); + } + return path.resolve(context.baseDir, filePath); +} + +function buildScriptGlobals( + env: Record, + output: Record, +): vm.Context { + return { + ...env, + output, + json: parseRunScriptJson, + http: { + post: (url: string, options?: { headers?: Record; body?: string }) => + runHttpRequestSync('POST', url, options), + }, + }; +} + +function parseRunScriptJson(value: unknown): unknown { + if (typeof value !== 'string') { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript json() expected a string body, received ${typeof value}.`, + ); + } + if (value.trim().length === 0) { + throw new AppError( + 'COMMAND_FAILED', + 'Maestro runScript json() received an empty body. Check the preceding http response status and setup server output.', + ); + } + try { + return JSON.parse(value, safeRunScriptJsonReviver) as unknown; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript json() could not parse response body: ${error instanceof Error ? error.message : String(error)}`, + { bodyPreview: value.slice(0, 1000) }, + error instanceof Error ? error : undefined, + ); + } +} + +function safeRunScriptJsonReviver(key: string, value: unknown): unknown { + return key === '__proto__' || key === 'constructor' || key === 'prototype' ? undefined : value; +} + +function runHttpRequestSync( + method: string, + url: string, + options?: { headers?: Record; body?: string }, +): HttpResponse { + // Keep http.post synchronous from the flow author's point of view while the + // network request remains timeout-bounded independently from node:vm. + const result = runCmdSync(process.execPath, ['-e', HTTP_REQUEST_SCRIPT], { + stdin: JSON.stringify({ + method, + url, + headers: options?.headers ?? {}, + body: options?.body ?? '', + }), + timeoutMs: RUN_SCRIPT_TIMEOUT_MS, + allowFailure: true, + }); + if (result.exitCode !== 0) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} failed for ${url}: ${trimHttpErrorOutput(result.stderr)}`, + { + exitCode: result.exitCode, + stderr: result.stderr, + }, + ); + } + try { + return JSON.parse(result.stdout) as HttpResponse; + } catch (error) { + throw new AppError( + 'COMMAND_FAILED', + `Maestro runScript http.${method.toLowerCase()} returned invalid JSON for ${url}`, + { + stdout: result.stdout.slice(0, 1000), + stderr: result.stderr.slice(0, 1000), + }, + error instanceof Error ? error : undefined, + ); + } +} + +function validateOutputKeys(output: Record, scriptPath: string): void { + for (const key of Object.keys(output)) { + if (!key.includes('.')) continue; + throw new AppError('INVALID_ARGS', `Maestro runScript output key cannot contain ".": ${key}`, { + scriptPath, + key, + }); + } +} + +function stringifyOutputValue(value: unknown): string { + if (typeof value === 'string') return value; + if (typeof value === 'number' || typeof value === 'boolean') return String(value); + return JSON.stringify(value); +} + +function trimHttpErrorOutput(stderr: string): string { + const trimmed = stderr.trim(); + return trimmed.length > 0 ? trimmed.slice(0, 1000) : 'request process exited without stderr'; +} diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts new file mode 100644 index 000000000..788434662 --- /dev/null +++ b/src/compat/maestro/runtime-assertions.ts @@ -0,0 +1,212 @@ +import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import type { DaemonResponse } from '../../daemon/types.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { + captureMaestroRawSnapshot, + errorResponse, + rememberMaestroSnapshot, + readSnapshotState, + type MaestroRuntimeInvoke, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + readMaestroSelectorPlatform, + resolveVisibleMaestroNodeFromSnapshot, +} from './runtime-targets.ts'; + +const MAESTRO_ASSERTION_POLICY = { + animationPollMs: 250, + assertVisibleGraceMs: 1000, + assertVisiblePollMs: 250, + assertNotVisiblePollMs: 250, + assertNotVisibleTimeoutMs: 3000, +} as const; + +export async function invokeMaestroAssertVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise { + const [selector, timeoutValue = '5000'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'assertVisible timeout must be a non-negative number.'); + } + + const startedAt = Date.now(); + const deadlineMs = timeoutMs + MAESTRO_ASSERTION_POLICY.assertVisibleGraceMs; + let lastResponse: DaemonResponse | undefined; + do { + const response = await captureMaestroRawSnapshot(params); + lastResponse = response; + if (response.ok) { + const snapshot = readSnapshotState(response.data); + if (!snapshot) { + return errorResponse('COMMAND_FAILED', 'Unable to read snapshot data for assertVisible.'); + } + const target = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + selector, + readMaestroSelectorPlatform(params.baseReq.flags), + getSnapshotReferenceFrame(snapshot), + ); + if (target.ok) { + rememberMaestroSnapshot(params.scope, response.data, selector); + return { + ok: true, + data: { + selector, + matches: target.matches, + nodeIndex: target.node.index, + nodeType: target.node.type, + nodeLabel: target.node.label, + nodeIdentifier: target.node.identifier, + rect: target.rect, + waitedMs: Date.now() - startedAt, + }, + }; + } + lastResponse = errorResponse('COMMAND_FAILED', target.message, { selector }); + } + + if (Date.now() - startedAt >= deadlineMs) break; + await sleep(MAESTRO_ASSERTION_POLICY.assertVisiblePollMs); + } while (Date.now() - startedAt <= deadlineMs); + + return ( + lastResponse ?? + errorResponse('COMMAND_FAILED', `Expected visible but did not match: ${selector}`, { + selector, + timeoutMs, + }) + ); +} + +export async function invokeMaestroAssertNotVisible(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'assertNotVisible requires a selector.'); + } + const startedAt = Date.now(); + let hiddenSamples = 0; + let lastVisibleResponse: DaemonResponse | undefined; + while (Date.now() - startedAt <= MAESTRO_ASSERTION_POLICY.assertNotVisibleTimeoutMs) { + const response = await params.invoke({ + ...params.baseReq, + command: 'is', + positionals: ['visible', selector], + flags: { ...params.baseReq.flags, noRecord: true }, + }); + if (response.ok) { + hiddenSamples = 0; + lastVisibleResponse = response; + } else if (isMaestroVisibilityMiss(response)) { + hiddenSamples += 1; + if (hiddenSamples >= 2) { + return { + ok: true, + data: { + pass: true, + selector, + stableSamples: hiddenSamples, + timeoutMs: MAESTRO_ASSERTION_POLICY.assertNotVisibleTimeoutMs, + }, + }; + } + } else { + return response; + } + await sleep(MAESTRO_ASSERTION_POLICY.assertNotVisiblePollMs); + } + return errorResponse('COMMAND_FAILED', `Expected not visible but matched: ${selector}`, { + selector, + timeoutMs: MAESTRO_ASSERTION_POLICY.assertNotVisibleTimeoutMs, + lastResponse: lastVisibleResponse, + }); +} + +export async function invokeMaestroWaitForAnimationToEnd(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const timeoutMs = Number(params.positionals[0] ?? 15000); + if (!Number.isFinite(timeoutMs) || timeoutMs < 0) { + return errorResponse('INVALID_ARGS', 'waitForAnimationToEnd timeout must be a number.'); + } + const startedAt = Date.now(); + let previousSignature: string | undefined; + let lastResponse: DaemonResponse | undefined; + + while (Date.now() - startedAt < timeoutMs) { + const response = await captureMaestroRawSnapshot(params); + const poll = readAnimationPollResult(response, previousSignature, timeoutMs); + if (poll.done) return poll.response; + previousSignature = poll.signature ?? previousSignature; + lastResponse = response; + await sleep(MAESTRO_ASSERTION_POLICY.animationPollMs); + } + + return lastResponse?.ok === false + ? lastResponse + : { ok: true, data: { stable: false, timeoutMs } }; +} + +function isMaestroVisibilityMiss(response: Extract): boolean { + const details = response.error.details; + return ( + details?.command === 'is' && + (details.reason === 'selector_not_found' || details.reason === 'predicate_failed') + ); +} + +function readAnimationPollResult( + response: DaemonResponse, + previousSignature: string | undefined, + timeoutMs: number, +): { done: true; response: DaemonResponse } | { done: false; signature?: string } { + const signature = readSnapshotStabilitySignature(response); + if (!response.ok) return { done: false }; + if (!signature) return { done: true, response }; + if (previousSignature === signature) { + return { done: true, response: { ok: true, data: { stable: true, timeoutMs } } }; + } + return { done: false, signature }; +} + +function readSnapshotStabilitySignature(response: DaemonResponse): string | null { + if (!response.ok) return null; + const snapshot = readSnapshotState(response.data); + return snapshot ? snapshotStabilitySignature(snapshot) : null; +} + +function snapshotStabilitySignature(snapshot: SnapshotState): string { + return JSON.stringify( + snapshot.nodes.map((node) => ({ + index: node.index, + parentIndex: node.parentIndex, + type: node.type, + identifier: node.identifier, + label: node.label, + value: node.value, + rect: node.rect + ? { + x: Math.round(node.rect.x), + y: Math.round(node.rect.y), + width: Math.round(node.rect.width), + height: Math.round(node.rect.height), + } + : undefined, + })), + ); +} diff --git a/src/compat/maestro/runtime-commands.ts b/src/compat/maestro/runtime-commands.ts new file mode 100644 index 000000000..d15292434 --- /dev/null +++ b/src/compat/maestro/runtime-commands.ts @@ -0,0 +1,14 @@ +export const MAESTRO_RUNTIME_COMMAND = { + runFlowWhen: '__maestroRunFlowWhen', + retry: '__maestroRetry', + runScript: '__maestroRunScript', + assertVisible: '__maestroAssertVisible', + assertNotVisible: '__maestroAssertNotVisible', + pressEnter: '__maestroPressEnter', + waitForAnimationToEnd: '__maestroWaitForAnimationToEnd', + scrollUntilVisible: '__maestroScrollUntilVisible', + swipeScreen: '__maestroSwipeScreen', + swipeOn: '__maestroSwipeOn', + tapOn: '__maestroTapOn', + tapPointPercent: '__maestroTapPointPercent', +} as const; diff --git a/src/compat/maestro/runtime-flow.ts b/src/compat/maestro/runtime-flow.ts new file mode 100644 index 000000000..4feb4f962 --- /dev/null +++ b/src/compat/maestro/runtime-flow.ts @@ -0,0 +1,157 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../../daemon/types.ts'; +import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import { + batchStepToSessionAction, + captureMaestroRawSnapshot, + errorResponse, + readSnapshotState, + type MaestroReplayInvoker, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + readMaestroSelectorPlatform, + resolveVisibleMaestroNodeFromSnapshot, +} from './runtime-targets.ts'; + +type MaestroRunFlowWhenCondition = + | { ok: true; mode: string; predicate: string; selector: string } + | { ok: false; response: DaemonResponse }; + +export async function invokeMaestroRunFlowWhen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const condition = readMaestroRunFlowWhenCondition(params.positionals); + if (!condition.ok) return condition.response; + const conditionResult = await evaluateMaestroRunFlowWhenCondition(params, condition); + if (!conditionResult.ok) return conditionResult.response; + if (!conditionResult.matched) { + return { + ok: true, + data: { skipped: true, condition: condition.mode, selector: condition.selector }, + }; + } + return await invokeMaestroRunFlowWhenSteps(params, condition); +} + +export async function invokeMaestroRetry(params: { + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + const [maxRetriesValue = '1'] = params.positionals; + const maxRetries = Number(maxRetriesValue); + if (!Number.isInteger(maxRetries) || maxRetries < 0) { + return errorResponse('INVALID_ARGS', 'retry.maxRetries must be a non-negative integer.'); + } + + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + let lastResponse: DaemonResponse | undefined; + for (let attempt = 0; attempt <= maxRetries; attempt += 1) { + const response = await invokeMaestroRetryAttempt(params, steps, attempt); + if (response.ok) { + return { ok: true, data: { attempts: attempt + 1, retried: attempt > 0 } }; + } + lastResponse = response; + } + return lastResponse ?? errorResponse('COMMAND_FAILED', 'retry commands failed.'); +} + +function readMaestroRunFlowWhenCondition(positionals: string[]): MaestroRunFlowWhenCondition { + const [mode, selector] = positionals; + if ((mode !== 'visible' && mode !== 'notVisible') || !selector) { + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'runFlow.when requires visible/notVisible and a selector.', + ), + }; + } + return { + ok: true, + mode, + predicate: mode === 'visible' ? 'visible' : 'hidden', + selector, + }; +} + +async function evaluateMaestroRunFlowWhenCondition( + params: { + baseReq: ReplayBaseRequest; + invoke: (req: DaemonRequest) => Promise; + }, + condition: Extract, +): Promise<{ ok: true; matched: boolean } | { ok: false; response: DaemonResponse }> { + const response = await captureMaestroRawSnapshot(params); + if (!response.ok) return { ok: false, response }; + const snapshot = readSnapshotState(response.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'Unable to read snapshot data for runFlow.when.'), + }; + } + const visible = resolveVisibleMaestroNodeFromSnapshot( + snapshot, + condition.selector, + readMaestroSelectorPlatform(params.baseReq.flags), + getSnapshotReferenceFrame(snapshot), + ).ok; + return { ok: true, matched: condition.mode === 'visible' ? visible : !visible }; +} + +async function invokeMaestroRunFlowWhenSteps( + params: { + batchSteps: CommandFlags['batchSteps'] | undefined; + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + condition: Extract, +): Promise { + const steps = (params.batchSteps ?? []).map(batchStepToSessionAction); + for (const [index, action] of steps.entries()) { + // Preserve stable parent-step ordering for nested runtime commands while + // keeping the substep distinguishable in traces. + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + index / 1000, + }); + if (!response.ok) return response; + } + + return { + ok: true, + data: { ran: steps.length, condition: condition.mode, selector: condition.selector }, + }; +} + +async function invokeMaestroRetryAttempt( + params: { + line: number; + step: number; + invokeReplayAction: MaestroReplayInvoker; + }, + steps: SessionAction[], + attempt: number, +): Promise { + for (const [index, action] of steps.entries()) { + const response = await params.invokeReplayAction({ + action, + line: params.line, + step: params.step + attempt + index / 1000, + }); + if (!response.ok) return response; + } + return { ok: true, data: { ran: steps.length } }; +} diff --git a/src/compat/maestro/runtime-geometry.ts b/src/compat/maestro/runtime-geometry.ts new file mode 100644 index 000000000..50396b0a5 --- /dev/null +++ b/src/compat/maestro/runtime-geometry.ts @@ -0,0 +1,129 @@ +import type { Rect, SnapshotNode } from '../../utils/snapshot.ts'; +import type { MaestroSnapshotTarget } from './runtime-targets.ts'; + +const MAESTRO_GEOMETRY_POLICY = { + swipe: { + screenRatio: 0.35, + minDistancePx: 120, + maxDistancePx: 360, + marginPx: 8, + }, + largeTextContainerBias: { + minWidth: 120, + minHeight: 70, + width: 168, + height: 48, + }, +} as const; + +export function swipeCoordinatesFromTarget( + target: MaestroSnapshotTarget, + direction: string, +): + | { ok: true; start: { x: number; y: number }; end: { x: number; y: number } } + | { ok: false; message: string } { + const center = pointInsideRect(target.rect); + const frame = target.frame; + const horizontalDistance = swipeDistance(frame?.referenceWidth, target.rect.width); + const verticalDistance = swipeDistance(frame?.referenceHeight, target.rect.height); + const margin = MAESTRO_GEOMETRY_POLICY.swipe.marginPx; + const minX = margin; + const minY = margin; + const maxX = frame ? frame.referenceWidth - margin : center.x + horizontalDistance; + const maxY = frame ? frame.referenceHeight - margin : center.y + verticalDistance; + switch (direction.toLowerCase()) { + case 'up': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y - verticalDistance, minY, maxY) }, + }; + case 'down': + return { + ok: true, + start: center, + end: { x: center.x, y: clampCoordinate(center.y + verticalDistance, minY, maxY) }, + }; + case 'left': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x - horizontalDistance, minX, maxX), y: center.y }, + }; + case 'right': + return { + ok: true, + start: center, + end: { x: clampCoordinate(center.x + horizontalDistance, minX, maxX), y: center.y }, + }; + default: + return { ok: false, message: 'swipe.label direction must be up, down, left, or right.' }; + } +} + +export function pointForMaestroTapOnTarget( + target: MaestroSnapshotTarget, + isVisibleTextSelector: boolean, +): { x: number; y: number } { + if (!shouldBiasMaestroVisibleTextTap(target.node, isVisibleTextSelector, target.rect)) { + return pointInsideRect(target.rect); + } + return { + x: interiorCoordinate( + target.rect.x, + Math.min(target.rect.width, MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.width), + ), + y: interiorCoordinate( + target.rect.y, + Math.min(target.rect.height, MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.height), + ), + }; +} + +function swipeDistance(frameSize: number | undefined, rectSize: number): number { + const screenRelative = + typeof frameSize === 'number' ? frameSize * MAESTRO_GEOMETRY_POLICY.swipe.screenRatio : 0; + return Math.round( + Math.min( + MAESTRO_GEOMETRY_POLICY.swipe.maxDistancePx, + Math.max(MAESTRO_GEOMETRY_POLICY.swipe.minDistancePx, screenRelative, rectSize * 1.5), + ), + ); +} + +function clampCoordinate(value: number, min: number, max: number): number { + return Math.round(Math.min(max, Math.max(min, value))); +} + +function pointInsideRect(rect: Rect): { x: number; y: number } { + return { + x: interiorCoordinate(rect.x, rect.width), + y: interiorCoordinate(rect.y, rect.height), + }; +} + +function shouldBiasMaestroVisibleTextTap( + node: SnapshotNode, + isVisibleTextSelector: boolean, + rect: Rect, +): boolean { + if (!isVisibleTextSelector) return false; + if ( + rect.height < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minHeight || + rect.width < MAESTRO_GEOMETRY_POLICY.largeTextContainerBias.minWidth + ) { + return false; + } + const type = node.type?.toLowerCase(); + return type === 'cell' || type === 'other' || type === 'scrollview'; +} + +function interiorCoordinate(origin: number, size: number): number { + // Maestro flows often expose hidden E2E controls as 1x1 views at the screen + // edge. Preserve zero-origin taps for those controls instead of nudging them + // outside their tiny rect by applying normal center/bounds clamping. + if (size <= 1) return Math.floor(origin); + const min = Math.ceil(origin); + const max = Math.floor(origin + size - 1); + return clampCoordinate(origin + size / 2, min, max); +} diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts new file mode 100644 index 000000000..8300b7d32 --- /dev/null +++ b/src/compat/maestro/runtime-interactions.ts @@ -0,0 +1,598 @@ +import { getSnapshotReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse } from '../../daemon/types.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import { sleep } from '../../utils/timeouts.ts'; +import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; +import { + captureMaestroRawSnapshot, + consumeMaestroSnapshot, + errorResponse, + readCachedMaestroReferenceFrame, + readSnapshotState, + type FailedDaemonResponse, + type MaestroRuntimeInvoke, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + extractMaestroVisibleTextQuery, + readMaestroSelectorPlatform, + resolveMaestroFuzzyTextNodeFromSnapshot, + resolveMaestroNodeFromSnapshot, + type MaestroSnapshotTarget, + type MaestroTapOnOptions, +} from './runtime-targets.ts'; + +const MAESTRO_INTERACTION_POLICY = { + scrollUntilVisibleProbeMs: 500, + tapOnRetryMs: 250, + tapOnTimeoutMs: 30000, + optionalTapOnTimeoutMs: 3000, +} as const; + +type MaestroScrollUntilVisibleParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}; + +type MaestroTapOnParams = { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}; + +export async function invokeMaestroScrollUntilVisible( + params: MaestroScrollUntilVisibleParams, +): Promise { + const [selector, timeoutValue = '5000', direction = 'down'] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible requires a selector.'); + } + const timeoutMs = Number(timeoutValue); + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + return errorResponse('INVALID_ARGS', 'scrollUntilVisible timeout must be a positive number.'); + } + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempts = Math.max( + 1, + Math.ceil(timeoutMs / MAESTRO_INTERACTION_POLICY.scrollUntilVisibleProbeMs), + ); + let lastWaitResponse: FailedDaemonResponse | null = null; + + for (let index = 0; index < attempts; index += 1) { + const probeResponse = await probeMaestroScrollVisibility( + params, + selector, + fuzzyTextQuery, + scrollProbeMs(timeoutMs, index), + ); + if (probeResponse.ok) return probeResponse; + lastWaitResponse = probeResponse; + + if (index === attempts - 1) break; + + const scrollResponse = await params.invoke({ + ...params.baseReq, + command: 'scroll', + positionals: [direction], + }); + if (!scrollResponse.ok) return scrollResponse; + } + + return withMaestroScrollTimeoutContext(lastWaitResponse, selector, timeoutMs); +} + +export async function invokeMaestroTapPointPercent(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const [xValue, yValue] = params.positionals; + const xPercent = Number(xValue); + const yPercent = Number(yValue); + if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) { + return errorResponse('INVALID_ARGS', 'tapOn percentage point requires numeric x/y values.'); + } + + const snapshotResponse = await captureMaestroRawSnapshot(params); + if (!snapshotResponse.ok) return snapshotResponse; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to read snapshot data for Maestro percentage point tap.', + ); + } + + const frame = getSnapshotReferenceFrame(snapshot); + if (!frame) { + return errorResponse( + 'COMMAND_FAILED', + 'Unable to resolve screen size for Maestro percentage point tap.', + ); + } + + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [ + String(Math.round((frame.referenceWidth * xPercent) / 100)), + String(Math.round((frame.referenceHeight * yPercent) / 100)), + ], + }); +} + +export async function invokeMaestroSwipeScreen(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise { + const swipe = await resolveMaestroScreenSwipe(params); + if (!swipe.ok) return swipe.response; + + return await invokeSwipeGesture(params, swipe, swipe.durationMs); +} + +export async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { + const [selector, rawOptions] = params.positionals; + if (!selector) { + return errorResponse('INVALID_ARGS', 'tapOn requires a selector.'); + } + const options = readMaestroTapOnOptions(rawOptions); + if (!options.ok) return options.response; + const startedAt = Date.now(); + const timeoutMs = maestroTapOnTimeoutMs(params); + let lastResponse: DaemonResponse | undefined; + while (Date.now() - startedAt < timeoutMs) { + const attempt = await attemptMaestroTapOn(params, selector, options.value ?? {}); + if (!attempt.retry) return attempt.response; + lastResponse = attempt.response; + await sleep(MAESTRO_INTERACTION_POLICY.tapOnRetryMs); + } + + return maestroTapOnTimeoutResponse(params, selector, lastResponse); +} + +export async function invokeMaestroSwipeOn(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; +}): Promise { + const [selector, direction = 'up', durationMs] = params.positionals; + if (!selector) return errorResponse('INVALID_ARGS', 'swipe.label requires a label selector.'); + const target = await resolveMaestroSnapshotTarget(params, selector, {}, 'swipe.label'); + if (!target.ok) return target.response; + const swipe = swipeCoordinatesFromTarget(target.target, direction); + if (!swipe.ok) return errorResponse('INVALID_ARGS', swipe.message); + return await invokeSwipeGesture(params, swipe, durationMs); +} + +async function invokeSwipeGesture( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + }, + swipe: { + start: { x: number; y: number }; + end: { x: number; y: number }; + }, + durationMs: string | undefined, +): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'swipe', + positionals: [ + String(swipe.start.x), + String(swipe.start.y), + String(swipe.end.x), + String(swipe.end.y), + ...(durationMs ? [durationMs] : []), + ], + }); +} + +async function resolveMaestroScreenSwipe(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise< + | { + ok: true; + start: { x: number; y: number }; + end: { x: number; y: number }; + durationMs?: string; + } + | { ok: false; response: DaemonResponse } +> { + const cachedFrame = readCachedMaestroReferenceFrame(params.scope); + const frame = cachedFrame ?? (await captureFrameForMaestroScreenSwipe(params)); + if (!frame) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'Unable to resolve screen size for Maestro swipe.'), + }; + } + + const [mode, ...args] = params.positionals; + if (mode === 'direction') return resolveDirectionalScreenSwipe(args, frame); + if (mode === 'percent') { + return resolvePercentScreenSwipe( + args, + frame, + readMaestroSelectorPlatform(params.baseReq.flags), + ); + } + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'Maestro screen swipe requires direction or percent.'), + }; +} + +async function captureFrameForMaestroScreenSwipe(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise<{ referenceWidth: number; referenceHeight: number } | undefined> { + const snapshotResponse = await captureMaestroRawSnapshot(params); + if (!snapshotResponse.ok) return undefined; + const snapshot = readSnapshotState(snapshotResponse.data); + return getSnapshotReferenceFrame(snapshot); +} + +function resolveDirectionalScreenSwipe( + args: string[], + frame: { referenceWidth: number; referenceHeight: number }, +): + | { + ok: true; + start: { x: number; y: number }; + end: { x: number; y: number }; + durationMs?: string; + } + | { ok: false; response: DaemonResponse } { + const [direction, durationMs] = args; + if (!direction) { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'Maestro direction swipe requires a direction.'), + }; + } + const point = (xPercent: number, yPercent: number) => percentPoint(frame, xPercent, yPercent, 8); + switch (direction) { + case 'up': + return { ok: true, start: point(50, 80), end: point(50, 20), durationMs }; + case 'down': + return { ok: true, start: point(50, 20), end: point(50, 80), durationMs }; + case 'left': + return { ok: true, start: point(80, 50), end: point(20, 50), durationMs }; + case 'right': + return { ok: true, start: point(20, 50), end: point(80, 50), durationMs }; + default: + return { + ok: false, + response: errorResponse( + 'INVALID_ARGS', + 'Maestro swipe direction must be UP, DOWN, LEFT, or RIGHT.', + ), + }; + } +} + +function resolvePercentScreenSwipe( + args: string[], + frame: { referenceWidth: number; referenceHeight: number }, + platform: string, +): + | { + ok: true; + start: { x: number; y: number }; + end: { x: number; y: number }; + durationMs?: string; + } + | { ok: false; response: DaemonResponse } { + const [startX, startY, endX, endY, durationMs] = args; + const values = [startX, startY, endX, endY].map(Number); + if (values.some((value) => !Number.isFinite(value))) { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'Maestro percentage swipe requires numeric points.'), + }; + } + const [x1, y1, x2, y2] = values as [number, number, number, number]; + const adjustedY = androidHorizontalContentSwipeY(platform, x1, y1, x2, y2); + return { + ok: true, + start: percentPoint(frame, x1, adjustedY, 1), + end: percentPoint(frame, x2, adjustedY, 1), + durationMs, + }; +} + +function androidHorizontalContentSwipeY( + platform: string, + x1: number, + y1: number, + x2: number, + y2: number, +): number { + if (platform !== 'android') return y2; + if (y1 !== y2 || y1 !== 50) return y2; + if (Math.abs(x2 - x1) < 50) return y2; + // Maestro's Android driver treats 50% horizontal swipes as content swipes. + // Raw `adb input swipe` at the physical screen midpoint can land above + // horizontally paged content in React Native layouts, so use a lower content + // lane for full-width horizontal Maestro percentage swipes. + return 65; +} + +function percentPoint( + frame: { referenceWidth: number; referenceHeight: number }, + xPercent: number, + yPercent: number, + marginPx: number, +): { x: number; y: number } { + return { + x: clampPoint( + Math.round((frame.referenceWidth * xPercent) / 100), + marginPx, + frame.referenceWidth, + ), + y: clampPoint( + Math.round((frame.referenceHeight * yPercent) / 100), + marginPx, + frame.referenceHeight, + ), + }; +} + +function clampPoint(value: number, marginPx: number, size: number): number { + const max = Math.max(marginPx, size - marginPx); + return Math.min(max, Math.max(marginPx, value)); +} + +async function probeMaestroScrollVisibility( + params: MaestroScrollUntilVisibleParams, + selector: string, + fuzzyTextQuery: string | null, + probeMs: number, +): Promise { + const waitResponse = await params.invoke({ + ...params.baseReq, + command: 'wait', + positionals: [selector, String(probeMs)], + }); + if (waitResponse.ok || !fuzzyTextQuery) return waitResponse; + + const fuzzyResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [fuzzyTextQuery, 'wait', String(probeMs)], + }); + return fuzzyResponse; +} + +function scrollProbeMs(timeoutMs: number, index: number): number { + return Math.min( + MAESTRO_INTERACTION_POLICY.scrollUntilVisibleProbeMs, + Math.max(1, timeoutMs - index * MAESTRO_INTERACTION_POLICY.scrollUntilVisibleProbeMs), + ); +} + +function maestroTapOnTimeoutMs(params: MaestroTapOnParams): number { + return params.baseReq.flags?.maestro?.optional === true + ? MAESTRO_INTERACTION_POLICY.optionalTapOnTimeoutMs + : MAESTRO_INTERACTION_POLICY.tapOnTimeoutMs; +} + +function maestroTapOnTimeoutResponse( + params: MaestroTapOnParams, + selector: string, + lastResponse: DaemonResponse | undefined, +): DaemonResponse { + if (params.baseReq.flags?.maestro?.optional === true) { + return { ok: true, data: { skipped: true, optional: true, selector } }; + } + return ( + lastResponse ?? errorResponse('COMMAND_FAILED', `tapOn timed out for selector: ${selector}`) + ); +} + +async function attemptMaestroTapOn( + params: MaestroTapOnParams, + selector: string, + options: MaestroTapOnOptions, +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + const attempt = await invokeMaestroSnapshotTapOn(params, selector, options); + if (attempt.response.ok) return { retry: false, response: attempt.response }; + if (attempt.targetResolved && fuzzyTextQuery) { + return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); + } + return { retry: true, response: attempt.response }; +} + +async function invokeMaestroSnapshotTapOn( + params: MaestroTapOnParams, + selector: string, + options: MaestroTapOnOptions, +): Promise<{ response: DaemonResponse; targetResolved: boolean }> { + const target = await resolveMaestroSnapshotTarget(params, selector, options, 'tapOn'); + if (!target.ok) return { response: target.response, targetResolved: false }; + const point = pointForMaestroTapOnTarget( + target.target, + extractMaestroVisibleTextQuery(selector) !== null, + ); + const response = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(point.x), String(point.y)], + }); + return { + response, + targetResolved: true, + }; +} + +async function invokeMaestroFuzzyTapOn( + params: MaestroTapOnParams, + query: string, +): Promise< + { retry: false; response: DaemonResponse } | { retry: true; response: FailedDaemonResponse } +> { + const findResponse = await params.invoke({ + ...params.baseReq, + command: 'find', + positionals: [query, 'click'], + flags: { + ...params.baseReq.flags, + findFirst: true, + }, + }); + if (findResponse.ok) return { retry: false, response: findResponse }; + return { retry: true, response: findResponse }; +} + +async function resolveMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; + }, + selector: string, + options: MaestroTapOnOptions, + commandLabel: string, +): Promise<{ ok: true; target: MaestroSnapshotTarget } | { ok: false; response: DaemonResponse }> { + const cachedTarget = resolveCachedMaestroSnapshotTarget(params, selector, options); + if (cachedTarget.ok) return cachedTarget; + + const snapshotResponse = await captureMaestroRawSnapshot(params); + if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; + + const snapshot = readSnapshotState(snapshotResponse.data); + if (!snapshot) { + return { + ok: false, + response: errorResponse( + 'COMMAND_FAILED', + `Unable to read snapshot data for ${commandLabel}.`, + ), + }; + } + + const frame = getSnapshotReferenceFrame(snapshot); + const resolution = resolveMaestroNodeFromSnapshot( + snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + frame, + ); + if (!resolution.ok) { + const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); + if (fuzzyTextQuery) { + const fuzzyResolution = resolveMaestroFuzzyTextNodeFromSnapshot( + snapshot, + fuzzyTextQuery, + frame, + ); + if (fuzzyResolution.ok) { + return { + ok: true, + target: { + node: fuzzyResolution.node, + rect: fuzzyResolution.rect, + frame, + }, + }; + } + } + } + if (!resolution.ok) { + return { + ok: false, + response: errorResponse('ELEMENT_NOT_FOUND', resolution.message, { + selector, + options, + command: commandLabel, + }), + }; + } + return { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame, + }, + }; +} + +function resolveCachedMaestroSnapshotTarget( + params: { + baseReq: ReplayBaseRequest; + scope?: ReplayVarScope; + }, + selector: string, + options: MaestroTapOnOptions, +): { ok: true; target: MaestroSnapshotTarget } | { ok: false } { + const cached = consumeMaestroSnapshot(params.scope, selector); + if (!cached) return { ok: false }; + const resolution = resolveMaestroNodeFromSnapshot( + cached.snapshot, + selector, + options, + readMaestroSelectorPlatform(params.baseReq.flags), + cached.frame, + ); + return resolution.ok + ? { + ok: true, + target: { + node: resolution.node, + rect: resolution.rect, + frame: cached.frame, + }, + } + : { ok: false }; +} + +function readMaestroTapOnOptions( + rawOptions: string | undefined, +): { ok: true; value: MaestroTapOnOptions | null } | { ok: false; response: DaemonResponse } { + if (!rawOptions) return { ok: true, value: null }; + try { + const value = JSON.parse(rawOptions) as MaestroTapOnOptions; + return { ok: true, value }; + } catch { + return { + ok: false, + response: errorResponse('INVALID_ARGS', 'tapOn runtime options must be valid JSON.'), + }; + } +} + +function withMaestroScrollTimeoutContext( + response: FailedDaemonResponse | null, + selector: string, + timeoutMs: number, +): DaemonResponse { + if (!response) { + return errorResponse( + 'COMMAND_FAILED', + `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}`, + ); + } + return { + ok: false, + error: { + ...response.error, + message: `scrollUntilVisible timed out after ${timeoutMs}ms for selector: ${selector}. Last wait: ${response.error.message}`, + }, + }; +} diff --git a/src/compat/maestro/runtime-support.ts b/src/compat/maestro/runtime-support.ts new file mode 100644 index 000000000..576da0ab7 --- /dev/null +++ b/src/compat/maestro/runtime-support.ts @@ -0,0 +1,120 @@ +import type { CommandFlags } from '../../core/dispatch.ts'; +import { + getSnapshotReferenceFrame, + type TouchReferenceFrame, +} from '../../daemon/touch-reference-frame.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../../daemon/types.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { SnapshotState } from '../../utils/snapshot.ts'; + +export type ReplayBaseRequest = Omit; + +export type MaestroReplayInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export type MaestroRuntimeInvoke = (req: DaemonRequest) => Promise; + +export type FailedDaemonResponse = Extract; + +const maestroReferenceFrameCache = new WeakMap(); +const maestroSnapshotCache = new WeakMap< + ReplayVarScope, + { snapshot: SnapshotState; frame: TouchReferenceFrame | undefined; selector: string } +>(); + +export function errorResponse( + code: string, + message: string, + details?: Record, +): FailedDaemonResponse { + return { + ok: false, + error: { code, message, ...(details ? { details } : {}) }, + }; +} + +export async function captureMaestroRawSnapshot(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + scope?: ReplayVarScope; +}): Promise { + const response = await params.invoke({ + ...params.baseReq, + command: 'snapshot', + positionals: [], + flags: { + ...params.baseReq.flags, + noRecord: true, + snapshotRaw: true, + snapshotForceFull: true, + }, + }); + if (response.ok && params.scope) rememberMaestroReferenceFrame(params.scope, response.data); + return response; +} + +export function readSnapshotState(data: unknown): SnapshotState | undefined { + if ( + typeof data === 'object' && + data !== null && + Array.isArray((data as { nodes?: unknown }).nodes) + ) { + return data as SnapshotState; + } + return undefined; +} + +export function readCachedMaestroReferenceFrame( + scope: ReplayVarScope | undefined, +): TouchReferenceFrame | undefined { + return scope ? maestroReferenceFrameCache.get(scope) : undefined; +} + +export function rememberMaestroSnapshot( + scope: ReplayVarScope | undefined, + data: unknown, + selector: string, +): void { + if (!scope) return; + const snapshot = readSnapshotState(data); + if (!snapshot) return; + maestroSnapshotCache.set(scope, { + snapshot, + frame: getSnapshotReferenceFrame(snapshot), + selector, + }); +} + +export function consumeMaestroSnapshot( + scope: ReplayVarScope | undefined, + selector: string, +): { snapshot: SnapshotState; frame: TouchReferenceFrame | undefined } | undefined { + if (!scope) return undefined; + const cached = maestroSnapshotCache.get(scope); + maestroSnapshotCache.delete(scope); + return cached?.selector === selector ? cached : undefined; +} + +function rememberMaestroReferenceFrame(scope: ReplayVarScope, data: unknown): void { + const snapshot = readSnapshotState(data); + const frame = getSnapshotReferenceFrame(snapshot); + if (frame) maestroReferenceFrameCache.set(scope, frame); +} + +export function batchStepToSessionAction( + step: NonNullable[number], +): SessionAction { + const action: SessionAction = { + ts: Date.now(), + command: step.command, + positionals: step.positionals ?? [], + flags: step.flags ?? {}, + }; + if (step.runtime && typeof step.runtime === 'object') { + action.runtime = step.runtime as SessionAction['runtime']; + } + return action; +} diff --git a/src/compat/maestro/runtime-targets.ts b/src/compat/maestro/runtime-targets.ts new file mode 100644 index 000000000..7170b5b03 --- /dev/null +++ b/src/compat/maestro/runtime-targets.ts @@ -0,0 +1,454 @@ +import type { Platform } from '../../utils/device.ts'; +import type { Rect, SnapshotNode, SnapshotState } from '../../utils/snapshot.ts'; +import { parseSelectorChain } from '../../daemon/selectors.ts'; +import { matchesSelector } from '../../daemon/selectors-match.ts'; +import { evaluateIsPredicate } from '../../utils/selector-is-predicates.ts'; +import { normalizeText } from '../../utils/finders.ts'; +import { extractNodeText } from '../../utils/snapshot-processing.ts'; +import type { TouchReferenceFrame } from '../../daemon/touch-reference-frame.ts'; +import type { DaemonRequest } from '../../daemon/types.ts'; +import type { Selector, SelectorTerm } from '../../daemon/selectors-parse.ts'; +import { detectReactNativeOverlay } from '../../commands/react-native/overlay.ts'; + +const MAESTRO_TAP_TARGET_TYPE_RANK = new Map([ + ['button', 0], + ['link', 0], + ['textfield', 0], + ['textview', 0], + ['searchfield', 0], + ['switch', 0], + ['slider', 0], + ['cell', 1], + ['statictext', 2], +]); + +export type MaestroTapOnOptions = { + childOf?: string; + index?: number; +}; + +export type MaestroSnapshotTarget = { + node: SnapshotNode; + rect: Rect; + frame?: TouchReferenceFrame; +}; + +type MaestroResolvedSnapshotMatch = { + node: SnapshotNode; + rect: Rect; + inheritedRect: boolean; +}; + +type ReactNativeOverlayFilterResult = { + matches: SnapshotNode[]; + blockedByReactNativeOverlay: boolean; +}; + +type SnapshotNodeByIndex = Map; + +export function resolveMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + options: MaestroTapOnOptions, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + let matches = findMaestroSelectorMatches(snapshot, selector, platform); + if (options.childOf) { + const parents = findMaestroSelectorMatches(snapshot, options.childOf, platform); + if (parents.length === 0) { + return { ok: false, message: `Maestro childOf parent did not match: ${options.childOf}` }; + } + const nodeByIndex = buildSnapshotNodeByIndex(snapshot.nodes); + matches = matches.filter((node) => + parents.some((parent) => + isDescendantOfSnapshotNode(snapshot.nodes, node, parent, nodeByIndex), + ), + ); + } + const filteredMatches = filterReactNativeOverlayBlockedMatches(snapshot.nodes, matches); + + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + filteredMatches.matches, + options.index, + extractMaestroVisibleTextQuery(selector), + frame, + ); + if (!target) { + const index = options.index ?? 0; + return { + ok: false, + message: filteredMatches.blockedByReactNativeOverlay + ? `Maestro selector matched ${matches.length} element(s), but React Native overlay is covering app content: ${selector}` + : `Maestro selector did not match index ${index}: ${selector}`, + }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +export function resolveMaestroFuzzyTextNodeFromSnapshot( + snapshot: SnapshotState, + query: string, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect } | { ok: false; message: string } { + const matches = findMaestroFuzzyTextMatches(snapshot, query); + const target = selectMaestroSnapshotMatch(snapshot.nodes, matches, undefined, query, frame); + if (!target) { + return { ok: false, message: `Maestro fuzzy text did not match: ${query}` }; + } + return { ok: true, node: target.node, rect: target.rect }; +} + +export function resolveVisibleMaestroNodeFromSnapshot( + snapshot: SnapshotState, + selector: string, + platform: Platform, + frame: TouchReferenceFrame | undefined, +): { ok: true; node: SnapshotNode; rect: Rect; matches: number } | { ok: false; message: string } { + const matches = findMaestroSelectorMatches(snapshot, selector, platform); + const visibleMatchesResult = filterVisibleMaestroMatches({ + nodes: snapshot.nodes, + matches, + platform, + }); + const target = selectMaestroSnapshotMatch( + snapshot.nodes, + visibleMatchesResult.matches, + undefined, + extractMaestroVisibleTextQuery(selector), + frame, + true, + ); + if (!target) { + return { + ok: false, + message: + matches.length > 0 + ? visibleMatchesResult.blockedByReactNativeOverlay + ? `Maestro selector matched ${matches.length} element(s), but React Native overlay is covering app content: ${selector}` + : `Maestro selector matched ${matches.length} element(s), but none were visible: ${selector}` + : `Maestro selector did not match: ${selector}`, + }; + } + return { + ok: true, + node: target.node, + rect: target.rect, + matches: visibleMatchesResult.matches.length, + }; +} + +function filterVisibleMaestroMatches(params: { + nodes: SnapshotState['nodes']; + matches: SnapshotNode[]; + platform: Platform; +}): { matches: SnapshotNode[]; blockedByReactNativeOverlay: boolean } { + const visibleMatches = params.matches.filter( + (node) => + evaluateIsPredicate({ + predicate: 'visible', + node, + nodes: params.nodes, + platform: params.platform, + }).pass, + ); + const overlayFilter = filterReactNativeOverlayBlockedMatches(params.nodes, visibleMatches); + return { + matches: overlayFilter.matches, + blockedByReactNativeOverlay: overlayFilter.blockedByReactNativeOverlay, + }; +} + +function filterReactNativeOverlayBlockedMatches( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], +): ReactNativeOverlayFilterResult { + const overlay = detectReactNativeOverlay(nodes); + if (!overlay.detected) { + return { matches, blockedByReactNativeOverlay: false }; + } + const overlayNodeIndexes = new Set( + [...overlay.dismissNodes, ...overlay.minimizeNodes, ...overlay.collapsedNodes].map( + (node) => node.index, + ), + ); + const overlayMatches = matches.filter((node) => overlayNodeIndexes.has(node.index)); + return { + matches: overlayMatches, + blockedByReactNativeOverlay: matches.length > 0 && overlayMatches.length === 0, + }; +} + +export function readMaestroSelectorPlatform(flags: DaemonRequest['flags']): Platform { + return flags?.platform === 'android' ? 'android' : 'ios'; +} + +export function extractMaestroVisibleTextQuery(selectorExpression: string): string | null { + const chain = parseSelectorChain(selectorExpression); + const terms = chain.selectors.flatMap((selector) => selector.terms); + if (terms.length === 0) return null; + // Mixed selectors may encode more than a visible-text lookup, so they keep + // the exact selector path instead of fuzzy text fallback. + if (!terms.some((term) => term.key === 'label' || term.key === 'text')) return null; + if (!terms.every((term) => ['label', 'text', 'id'].includes(term.key))) return null; + const values = terms.map((term) => (typeof term.value === 'string' ? term.value : '')); + const first = values[0]; + if (!first || !values.every((value) => value === first)) return null; + return first; +} + +function findMaestroSelectorMatches( + snapshot: SnapshotState, + selectorExpression: string, + platform: Platform, +): SnapshotNode[] { + const chain = parseSelectorChain(selectorExpression); + for (const selector of chain.selectors) { + const matches = snapshot.nodes.filter((node) => + matchesMaestroSelector(node, selector, platform), + ); + if (matches.length > 0) return matches; + } + return []; +} + +function findMaestroFuzzyTextMatches(snapshot: SnapshotState, query: string): SnapshotNode[] { + const normalizedQuery = normalizeText(query); + if (!normalizedQuery) return []; + const exact: SnapshotNode[] = []; + const partial: SnapshotNode[] = []; + for (const node of snapshot.nodes) { + const values = [node.label, extractNodeText(node), node.identifier, node.value].filter( + (value): value is string => Boolean(value), + ); + const normalizedValues = values.map((value) => normalizeText(value)); + if (normalizedValues.some((value) => value === normalizedQuery)) { + exact.push(node); + } else if (normalizedValues.some((value) => value.includes(normalizedQuery))) { + partial.push(node); + } + } + return exact.length > 0 ? exact : partial; +} + +function matchesMaestroSelector( + node: SnapshotNode, + selector: Selector, + platform: Platform, +): boolean { + if (matchesSelector(node, selector, platform)) return true; + return selector.terms.every((term) => matchesMaestroTerm(node, term, platform)); +} + +function matchesMaestroTerm(node: SnapshotNode, term: SelectorTerm, platform: Platform): boolean { + if (typeof term.value !== 'string' || !isMaestroRegexTextKey(term.key)) { + return matchesSelector(node, { raw: term.key, terms: [term] }, platform); + } + const value = readMaestroTextTermValue(node, term.key); + return textEqualsOrRegex(value, term.value); +} + +function isMaestroRegexTextKey(key: SelectorTerm['key']): key is 'id' | 'label' | 'text' | 'value' { + return key === 'id' || key === 'label' || key === 'text' || key === 'value'; +} + +function readMaestroTextTermValue( + node: SnapshotNode, + key: 'id' | 'label' | 'text' | 'value', +): string | undefined { + if (key === 'id') return node.identifier; + if (key === 'label') return node.label; + if (key === 'value') return node.value; + return extractNodeText(node); +} + +function textEqualsOrRegex(value: string | undefined, query: string): boolean { + const text = value ?? ''; + if (normalizeText(text) === normalizeText(query)) return true; + try { + return new RegExp(query).test(text); + } catch { + return false; + } +} + +function resolveNodeRect( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): { rect: Rect; inherited: boolean } | null { + if (node.rect && node.rect.width > 0 && node.rect.height > 0) { + return { rect: node.rect, inherited: false }; + } + if (node.rect) return null; + const rect = resolveRectlessNodeAncestorRect(nodes, node, nodeByIndex); + return rect ? { rect, inherited: true } : null; +} + +function resolveRectlessNodeAncestorRect( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): Rect | null { + let current: SnapshotNode | undefined = node; + while (typeof current.parentIndex === 'number') { + current = nodeByIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + if (!current.rect) continue; + return current.rect.width > 0 && current.rect.height > 0 ? current.rect : null; + } + return null; +} + +function selectMaestroSnapshotMatch( + nodes: SnapshotState['nodes'], + matches: SnapshotNode[], + index: number | undefined, + visibleTextQuery: string | null, + frame: TouchReferenceFrame | undefined, + requireOnScreen = false, +): { node: SnapshotNode; rect: Rect } | null { + const nodeByIndex = buildSnapshotNodeByIndex(nodes); + const resolved = matches + .map((node) => { + const match = resolveNodeRect(nodes, node, nodeByIndex); + return match ? { node, rect: match.rect, inheritedRect: match.inherited } : null; + }) + .filter((candidate): candidate is MaestroResolvedSnapshotMatch => Boolean(candidate)); + const candidates = + visibleTextQuery && index === undefined + ? preferOnScreenMatches(resolved, frame, requireOnScreen) + : resolved; + if (index !== undefined) { + return candidates[index] ?? null; + } + const sorted = candidates.sort((left, right) => + compareMaestroSnapshotMatches(left, right, visibleTextQuery), + ); + return sorted[0] ?? null; +} + +function preferOnScreenMatches( + matches: MaestroResolvedSnapshotMatch[], + frame: TouchReferenceFrame | undefined, + requireOnScreen: boolean, +): MaestroResolvedSnapshotMatch[] { + const onScreen = matches.filter((match) => isRectOnScreen(match.rect, frame)); + if (requireOnScreen) return onScreen; + return onScreen.length > 0 ? onScreen : matches; +} + +function isRectOnScreen(rect: Rect, frame: TouchReferenceFrame | undefined): boolean { + const maxX = frame?.referenceWidth ?? Number.POSITIVE_INFINITY; + const maxY = frame?.referenceHeight ?? Number.POSITIVE_INFINITY; + return rect.x < maxX && rect.y < maxY && rect.x + rect.width > 0 && rect.y + rect.height > 0; +} + +function compareMaestroSnapshotMatches( + left: MaestroResolvedSnapshotMatch, + right: MaestroResolvedSnapshotMatch, + visibleTextQuery: string | null, +): number { + const priorityRank = compareMaestroSnapshotMatchPriority(left, right, visibleTextQuery); + if (priorityRank !== 0) return priorityRank; + + if (!sameRoundedRect(left.rect, right.rect)) { + return left.node.index - right.node.index; + } + + const depthRank = (right.node.depth ?? 0) - (left.node.depth ?? 0); + if (depthRank !== 0) return depthRank; + + // Android transparent stacks can expose both the background screen and the + // foreground screen at the same coordinates. UIAutomator reports the + // foreground duplicate later in the snapshot, which matches Maestro's + // practical tap target for overlapping duplicates. + return right.node.index - left.node.index; +} + +function sameRoundedRect(left: Rect, right: Rect): boolean { + return ( + Math.round(left.x) === Math.round(right.x) && + Math.round(left.y) === Math.round(right.y) && + Math.round(left.width) === Math.round(right.width) && + Math.round(left.height) === Math.round(right.height) + ); +} + +function compareMaestroSnapshotMatchPriority( + left: MaestroResolvedSnapshotMatch, + right: MaestroResolvedSnapshotMatch, + visibleTextQuery: string | null, +): number { + if (visibleTextQuery) { + const textRank = + maestroVisibleTextMatchRank(left.node, visibleTextQuery) - + maestroVisibleTextMatchRank(right.node, visibleTextQuery); + if (textRank !== 0) return textRank; + } + + const typeRank = maestroTapTargetTypeRank(left.node) - maestroTapTargetTypeRank(right.node); + if (typeRank !== 0) return typeRank; + + const rectSourceRank = Number(left.inheritedRect) - Number(right.inheritedRect); + if (rectSourceRank !== 0) return rectSourceRank; + + const areaRank = + visibleTextQuery && maestroTapTargetTypeRank(left.node) === maestroTapTargetTypeRank(right.node) + ? rectArea(right.rect) - rectArea(left.rect) + : rectArea(left.rect) - rectArea(right.rect); + if (areaRank !== 0) return areaRank; + return 0; +} + +function rectArea(rect: Rect): number { + return rect.width * rect.height; +} + +function maestroTapTargetTypeRank(node: SnapshotNode): number { + return MAESTRO_TAP_TARGET_TYPE_RANK.get(node.type?.toLowerCase() ?? '') ?? 3; +} + +function maestroVisibleTextMatchRank(node: SnapshotNode, query: string): number { + const values = [node.label, extractNodeText(node), node.identifier, node.value].filter( + (value): value is string => Boolean(value), + ); + if (values.some((value) => value === query)) return 0; + if (values.some((value) => normalizeText(value) === normalizeText(query))) return 1; + if (values.some((value) => textEqualsOrRegex(value, query))) return 2; + return 3; +} + +function isDescendantOfSnapshotNode( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + ancestor: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, +): boolean { + return Boolean( + findSnapshotAncestor(nodes, node, nodeByIndex, (candidate) => + candidate === ancestor || candidate.index === ancestor.index ? candidate : null, + ), + ); +} + +function findSnapshotAncestor( + nodes: SnapshotState['nodes'], + node: SnapshotNode, + nodeByIndex: SnapshotNodeByIndex, + resolve: (ancestor: SnapshotNode) => T | null, +): T | null { + let current: SnapshotNode | undefined = node; + while (typeof current.parentIndex === 'number') { + current = nodeByIndex.get(current.parentIndex) ?? nodes[current.parentIndex]; + if (!current) return null; + const result = resolve(current); + if (result) return result; + } + return null; +} + +function buildSnapshotNodeByIndex(nodes: SnapshotState['nodes']): SnapshotNodeByIndex { + return new Map(nodes.map((candidate) => [candidate.index, candidate])); +} diff --git a/src/compat/maestro/runtime.ts b/src/compat/maestro/runtime.ts new file mode 100644 index 000000000..42cd0950f --- /dev/null +++ b/src/compat/maestro/runtime.ts @@ -0,0 +1,108 @@ +import { type CommandFlags } from '../../core/dispatch.ts'; +import { asAppError } from '../../utils/errors.ts'; +import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { DaemonRequest, DaemonResponse } from '../../daemon/types.ts'; +import { executeRunScriptFile } from './run-script.ts'; +import { MAESTRO_RUNTIME_COMMAND } from './runtime-commands.ts'; +import { + invokeMaestroAssertNotVisible, + invokeMaestroAssertVisible, + invokeMaestroWaitForAnimationToEnd, +} from './runtime-assertions.ts'; +import { invokeMaestroRetry, invokeMaestroRunFlowWhen } from './runtime-flow.ts'; +import { + errorResponse, + type MaestroReplayInvoker, + type MaestroRuntimeInvoke, + type ReplayBaseRequest, +} from './runtime-support.ts'; +import { + invokeMaestroScrollUntilVisible, + invokeMaestroSwipeScreen, + invokeMaestroSwipeOn, + invokeMaestroTapOn, + invokeMaestroTapPointPercent, +} from './runtime-interactions.ts'; + +export async function invokeMaestroRuntimeCommand(params: { + command: string; + baseReq: ReplayBaseRequest; + positionals: string[]; + batchSteps: CommandFlags['batchSteps'] | undefined; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: MaestroReplayInvoker; +}): Promise { + switch (params.command) { + case MAESTRO_RUNTIME_COMMAND.assertVisible: + return await invokeMaestroAssertVisible(params); + case MAESTRO_RUNTIME_COMMAND.assertNotVisible: + return await invokeMaestroAssertNotVisible(params); + case MAESTRO_RUNTIME_COMMAND.retry: + return await invokeMaestroRetry(params); + case MAESTRO_RUNTIME_COMMAND.pressEnter: + return await invokeMaestroPressEnter(params); + case MAESTRO_RUNTIME_COMMAND.waitForAnimationToEnd: + return await invokeMaestroWaitForAnimationToEnd(params); + case MAESTRO_RUNTIME_COMMAND.scrollUntilVisible: + return await invokeMaestroScrollUntilVisible(params); + case MAESTRO_RUNTIME_COMMAND.swipeScreen: + return await invokeMaestroSwipeScreen(params); + case MAESTRO_RUNTIME_COMMAND.swipeOn: + return await invokeMaestroSwipeOn(params); + case MAESTRO_RUNTIME_COMMAND.tapOn: + return await invokeMaestroTapOn(params); + case MAESTRO_RUNTIME_COMMAND.tapPointPercent: + return await invokeMaestroTapPointPercent(params); + case MAESTRO_RUNTIME_COMMAND.runFlowWhen: + return await invokeMaestroRunFlowWhen(params); + case MAESTRO_RUNTIME_COMMAND.runScript: + return invokeMaestroRunScript(params); + default: + return undefined; + } +} + +async function invokeMaestroPressEnter(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; +}): Promise { + const keyboardResponse = await params.invoke({ + ...params.baseReq, + command: 'keyboard', + positionals: ['enter'], + }); + if (keyboardResponse.ok) return keyboardResponse; + + return await params.invoke({ + ...params.baseReq, + command: 'type', + positionals: ['\n'], + }); +} + +function invokeMaestroRunScript(params: { + baseReq: ReplayBaseRequest; + positionals: string[]; + scope: ReplayVarScope; +}): DaemonResponse { + const [scriptPath] = params.positionals; + if (!scriptPath) { + return errorResponse('INVALID_ARGS', 'runScript requires a file path.'); + } + try { + const outputEnv = executeRunScriptFile({ + scriptPath, + env: { + ...params.scope.values, + ...(params.baseReq.flags?.maestro?.runScriptEnv ?? {}), + }, + }); + return { ok: true, data: { outputEnv } }; + } catch (error) { + const appError = asAppError(error); + return errorResponse(appError.code, appError.message, appError.details); + } +} diff --git a/src/compat/maestro/support.ts b/src/compat/maestro/support.ts index 3bd998faf..39d3c20b8 100644 --- a/src/compat/maestro/support.ts +++ b/src/compat/maestro/support.ts @@ -52,35 +52,6 @@ export function normalizePlatform(value: string | undefined): 'android' | 'ios' return normalizePlatformName(value); } -export function normalizePlatformValue(value: unknown, name: string): 'android' | 'ios' { - if (typeof value !== 'string') { - throw new AppError('INVALID_ARGS', `${name} expects Android or iOS.`); - } - const platform = normalizePlatformName(value); - if (!platform) { - throw new AppError('INVALID_ARGS', `${name} expects Android or iOS.`); - } - return platform; -} - -export function normalizeToken(value: string): string { - return value - .trim() - .replace(/([a-z0-9])([A-Z])/g, '$1-$2') - .replace(/[\s_]+/g, '-') - .toLowerCase(); -} - -export function readBooleanLiteral(value: unknown, command: string): boolean { - if (typeof value === 'boolean') return value; - if (typeof value === 'string') { - const normalized = normalizeToken(value); - if (normalized === 'true') return true; - if (normalized === 'false') return false; - } - throw new AppError('INVALID_ARGS', `${command} expects a boolean value.`); -} - export function readEnvMap(value: unknown, name: string): Record { if (value === undefined || value === null) return {}; if (!isPlainRecord(value)) { @@ -113,15 +84,11 @@ export function requireStringValue(command: string, value: unknown): string { } export function resolveMaestroString(value: string, context: MaestroParseContext): string { - return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (match, key: string) => { + return value.replace(/\$\{([A-Za-z_][A-Za-z0-9_.]*)\}/g, (match, key: string) => { return Object.prototype.hasOwnProperty.call(context.env, key) ? context.env[key] : match; }); } -export function resolveMaybeMaestroString(value: unknown, context: MaestroParseContext): unknown { - return typeof value === 'string' ? resolveMaestroString(value, context) : value; -} - export function unsupportedCommand(command: string): never { throw unsupportedMaestroSyntax(`Maestro command "${command}" is not supported yet.`); } diff --git a/src/compat/maestro/types.ts b/src/compat/maestro/types.ts index 81012d5e7..8ea39be92 100644 --- a/src/compat/maestro/types.ts +++ b/src/compat/maestro/types.ts @@ -31,5 +31,3 @@ export type MaestroParseContext = { export type MaestroCommandMapperDeps = { parseRunFlowFile(filePath: string, context: MaestroParseContext): MaestroReplayFlow; }; - -export type PermissionCommand = 'grant' | 'deny' | 'reset'; diff --git a/src/core/__tests__/app-events.test.ts b/src/core/__tests__/app-events.test.ts index aa8c4e89d..a2ebee940 100644 --- a/src/core/__tests__/app-events.test.ts +++ b/src/core/__tests__/app-events.test.ts @@ -9,4 +9,3 @@ test('parseTriggerAppEventArgs validates event name format', () => { (error) => error instanceof AppError && error.code === 'INVALID_ARGS', ); }); - diff --git a/src/core/__tests__/dispatch-interactions.test.ts b/src/core/__tests__/dispatch-interactions.test.ts index eba5b2eff..e5f4716f1 100644 --- a/src/core/__tests__/dispatch-interactions.test.ts +++ b/src/core/__tests__/dispatch-interactions.test.ts @@ -5,10 +5,7 @@ import { handleTransformGestureCommand, } from '../dispatch-interactions.ts'; import type { Interactor } from '../interactor-types.ts'; -import { - ANDROID_EMULATOR, - IOS_SIMULATOR, -} from '../../__tests__/test-utils/device-fixtures.ts'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; vi.mock('../../platforms/ios/macos-helper.ts', async (importOriginal) => { const actual = await importOriginal(); diff --git a/src/core/__tests__/dispatch-keyboard.test.ts b/src/core/__tests__/dispatch-keyboard.test.ts new file mode 100644 index 000000000..8cd0a8c2e --- /dev/null +++ b/src/core/__tests__/dispatch-keyboard.test.ts @@ -0,0 +1,48 @@ +import { beforeEach, test, vi } from 'vitest'; +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; + +vi.mock('../../platforms/ios/runner-client.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, runIosRunnerCommand: vi.fn() }; +}); + +import { dispatchCommand } from '../dispatch.ts'; +import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts'; +import { ANDROID_EMULATOR, IOS_DEVICE } from '../../__tests__/test-utils/device-fixtures.ts'; +import { withMockedAdb } from '../../__tests__/test-utils/mocked-binaries.ts'; + +const mockRunIosRunnerCommand = vi.mocked(runIosRunnerCommand); + +beforeEach(() => { + vi.resetAllMocks(); + mockRunIosRunnerCommand.mockResolvedValue({ + message: 'keyboardReturn', + wasVisible: true, + visible: false, + }); +}); + +test('dispatch keyboard enter sends Android ENTER keyevent', async () => { + await withMockedAdb('agent-device-dispatch-keyboard-enter-', async (argsLogPath) => { + const result = await dispatchCommand(ANDROID_EMULATOR, 'keyboard', ['enter']); + + assert.equal(result?.action, 'enter'); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\ninput\nkeyevent\nENTER/); + }); +}); + +test('dispatch keyboard enter sends native iOS keyboard return command', async () => { + const result = await dispatchCommand(IOS_DEVICE, 'keyboard', ['return'], undefined, { + appBundleId: 'com.example.app', + }); + + assert.equal(result?.action, 'enter'); + assert.equal(result?.wasVisible, true); + assert.equal(mockRunIosRunnerCommand.mock.calls.length, 1); + assert.deepEqual(mockRunIosRunnerCommand.mock.calls[0]?.[1], { + command: 'keyboardReturn', + appBundleId: 'com.example.app', + }); +}); diff --git a/src/core/__tests__/dispatch-open.test.ts b/src/core/__tests__/dispatch-open.test.ts index c55cbbe68..1cf399a73 100644 --- a/src/core/__tests__/dispatch-open.test.ts +++ b/src/core/__tests__/dispatch-open.test.ts @@ -1,8 +1,30 @@ -import { test } from 'vitest'; +import { beforeEach, test, vi } from 'vitest'; import assert from 'node:assert/strict'; import { dispatchCommand } from '../dispatch.ts'; import { AppError } from '../../utils/errors.ts'; import type { DeviceInfo } from '../../utils/device.ts'; +import { clearIosSimulatorAppState, openIosApp } from '../../platforms/ios/apps.ts'; +import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; + +vi.mock('../../platforms/ios/apps.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + clearIosSimulatorAppState: vi.fn(async () => ({ + bundleId: 'com.example.app', + containerPath: '/tmp/com.example.app', + })), + openIosApp: vi.fn(async () => {}), + }; +}); + +const mockClearIosSimulatorAppState = vi.mocked(clearIosSimulatorAppState); +const mockOpenIosApp = vi.mocked(openIosApp); + +beforeEach(() => { + mockClearIosSimulatorAppState.mockClear(); + mockOpenIosApp.mockClear(); +}); test('dispatch open rejects URL as first argument when second URL is provided', async () => { const device: DeviceInfo = { @@ -23,3 +45,47 @@ test('dispatch open rejects URL as first argument when second URL is provided', }, ); }); + +test('dispatch open rejects Android launch arguments instead of dropping them', async () => { + const device: DeviceInfo = { + platform: 'android', + id: 'emulator-5554', + name: 'Pixel', + kind: 'emulator', + booted: true, + }; + + await assert.rejects( + () => + dispatchCommand(device, 'open', ['com.example.app'], undefined, { + launchArgs: ['--fixture', 'demo'], + }), + (error: unknown) => { + assert.equal(error instanceof AppError, true); + assert.equal((error as AppError).code, 'UNSUPPORTED_OPERATION'); + assert.match((error as AppError).message, /Apple platforms/i); + return true; + }, + ); +}); + +test('dispatch open clears Maestro iOS simulator state and launches once', async () => { + const result = await dispatchCommand(IOS_SIMULATOR, 'open', ['com.example.app'], undefined, { + clearAppState: true, + launchArgs: ['-EXDevMenuIsOnboardingFinished', 'true'], + }); + + assert.equal(result?.app, 'com.example.app'); + assert.equal(mockClearIosSimulatorAppState.mock.calls.length, 1); + assert.deepEqual(mockClearIosSimulatorAppState.mock.calls[0]?.slice(0, 2), [ + IOS_SIMULATOR, + 'com.example.app', + ]); + assert.equal(mockOpenIosApp.mock.calls.length, 1); + assert.equal(mockOpenIosApp.mock.calls[0]?.[0], IOS_SIMULATOR); + assert.equal(mockOpenIosApp.mock.calls[0]?.[1], 'com.example.app'); + assert.deepEqual(mockOpenIosApp.mock.calls[0]?.[2]?.launchArgs, [ + '-EXDevMenuIsOnboardingFinished', + 'true', + ]); +}); diff --git a/src/core/__tests__/dispatch-series.test.ts b/src/core/__tests__/dispatch-series.test.ts index b8b5e0b18..e7630d63f 100644 --- a/src/core/__tests__/dispatch-series.test.ts +++ b/src/core/__tests__/dispatch-series.test.ts @@ -93,4 +93,3 @@ test('shouldUseIosDragSeries returns false when count is 1', () => { // --- computeDeterministicJitter --- // --- runRepeatedSeries --- - diff --git a/src/core/capabilities.ts b/src/core/capabilities.ts index b96573010..8ff7bfeb7 100644 --- a/src/core/capabilities.ts +++ b/src/core/capabilities.ts @@ -96,7 +96,7 @@ const COMMAND_CAPABILITY_MATRIX: Record = { device.kind === 'simulator', }, keyboard: { - // iOS only supports keyboard dismiss; status/get remains Android-only. + // iOS only supports keyboard dismiss/enter; status/get remains Android-only. apple: { simulator: true, device: true }, android: { emulator: true, device: true, unknown: true }, linux: LINUX_NONE, diff --git a/src/core/dispatch-context.ts b/src/core/dispatch-context.ts index 37387f42b..27b5b2f59 100644 --- a/src/core/dispatch-context.ts +++ b/src/core/dispatch-context.ts @@ -10,8 +10,17 @@ export type BatchStep = { runtime?: unknown; }; +export type MaestroRuntimeFlags = { + allowNonHittableCoordinateFallback?: boolean; + optional?: boolean; + runScriptEnv?: Record; +}; + export type CommandFlags = Omit & { batchSteps?: BatchStep[]; + clearAppState?: boolean; + launchArgs?: string[]; + maestro?: MaestroRuntimeFlags; replayBackend?: string; }; @@ -20,6 +29,8 @@ export type DispatchContext = ScreenshotDispatchFlags & { appBundleId?: string; activity?: string; launchConsole?: string; + launchArgs?: string[]; + clearAppState?: boolean; verbose?: boolean; logPath?: string; traceLogPath?: string; @@ -44,5 +55,6 @@ export type DispatchContext = ScreenshotDispatchFlags & { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableCoordinateFallback?: boolean; }; }; diff --git a/src/core/dispatch-interactions.ts b/src/core/dispatch-interactions.ts index 530862e7f..76b78fa64 100644 --- a/src/core/dispatch-interactions.ts +++ b/src/core/dispatch-interactions.ts @@ -86,6 +86,15 @@ export async function handleFillCommand( positionals: string[], context: DispatchContext | undefined, ): Promise> { + if (context?.directElementSelector) { + return await handleDirectElementSelectorFill( + interactor, + context.directElementSelector, + positionals, + context, + ); + } + const x = Number(positionals[0]); const y = Number(positionals[1]); const text = positionals.slice(2).join(' '); @@ -97,6 +106,28 @@ export async function handleFillCommand( return { x, y, text, delayMs, ...successText(formatTextLengthMessage('Filled', text)) }; } +async function handleDirectElementSelectorFill( + interactor: Interactor, + selector: NonNullable, + positionals: string[], + context: DispatchContext, +): Promise> { + if (!interactor.fillElementSelector) { + throw new AppError('UNSUPPORTED_OPERATION', 'direct element selector fill is not supported'); + } + const text = positionals.join(' '); + if (!text) throw new AppError('INVALID_ARGS', 'fill requires text'); + const delayMs = requireIntInRange(context.delayMs ?? 0, 'delay-ms', 0, 10_000); + const result = await interactor.fillElementSelector(selector, text, delayMs); + return { + selector: selector.raw, + text, + delayMs, + ...(result ?? {}), + ...successText(formatTextLengthMessage('Filled', text)), + }; +} + export async function handlePressCommand( device: DeviceInfo, interactor: Interactor, diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index f28a3c6e8..4367157ab 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -6,11 +6,12 @@ import { dismissAndroidKeyboard, getAndroidKeyboardState, } from '../platforms/android/device-input-state.ts'; +import { pressAndroidEnter } from '../platforms/android/input-actions.ts'; import { pushAndroidNotification } from '../platforms/android/notifications.ts'; import { getInteractor } from './interactors.ts'; import type { Interactor, RunnerContext } from './interactor-types.ts'; import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; -import { pushIosNotification } from '../platforms/ios/apps.ts'; +import { clearIosSimulatorAppState, pushIosNotification } from '../platforms/ios/apps.ts'; import { isDeepLinkTarget } from './open-target.ts'; import { parseTriggerAppEventArgs, resolveAppEventUrl } from './app-events.ts'; import { @@ -21,6 +22,7 @@ import { emitDiagnostic, withDiagnosticTimer } from '../utils/diagnostics.ts'; import { readLocationCoordinate } from '../utils/location-coordinates.ts'; import { successText, withSuccessText } from '../utils/success-text.ts'; import { screenshotOptionsFromFlags } from '../commands/capture-screenshot-options.ts'; +import { isKeyboardAction, type KeyboardAction } from '../utils/keyboard-actions.ts'; import type { DispatchContext } from './dispatch-context.ts'; import { handleFillCommand, @@ -43,7 +45,6 @@ import { parseDeviceRotation } from './device-rotation.ts'; export { resolveTargetDevice } from './dispatch-resolve.ts'; export type { BatchStep, CommandFlags, DispatchContext } from './dispatch-context.ts'; -// fallow-ignore-next-line complexity export async function dispatchCommand( device: DeviceInfo, command: string, @@ -71,98 +72,15 @@ export async function dispatchCommand( return await withDiagnosticTimer( 'platform_command', async () => { - switch (command) { - case 'open': - return handleOpenCommand(device, interactor, positionals, context); - case 'close': { - const app = positionals[0]; - if (!app) { - return { closed: 'session', ...successText('Closed session') }; - } - await interactor.close(app); - return { app, ...successText(`Closed: ${app}`) }; - } - case 'press': - return handlePressCommand(device, interactor, positionals, context); - case 'swipe': - return handleSwipeCommand(device, interactor, positionals, context); - case 'pan': - return handlePanCommand(interactor, positionals); - case 'fling': - return handleFlingCommand(interactor, positionals); - case 'longpress': - return handleLongPressCommand(interactor, positionals); - case 'focus': - return handleFocusCommand(interactor, positionals); - case 'type': - return handleTypeCommand(interactor, positionals, context); - case 'fill': - return handleFillCommand(interactor, positionals, context); - case 'scroll': - return handleScrollCommand(interactor, positionals, context); - case 'pinch': - return handlePinchCommand(device, interactor, positionals, context); - case 'rotate-gesture': - return handleRotateGestureCommand(device, interactor, positionals); - case 'transform-gesture': - return handleTransformGestureCommand(device, interactor, positionals); - case 'trigger-app-event': { - const { eventName, payload } = parseTriggerAppEventArgs(positionals); - const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); - await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); - return { - event: eventName, - eventUrl, - transport: 'deep-link', - ...successText(`Triggered app event: ${eventName}`), - }; - } - case 'screenshot': { - const positionalPath = positionals[0]; - const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; - await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); - const screenshotOptions = screenshotOptionsFromFlags(context); - await interactor.screenshot(screenshotPath, { - appBundleId: context?.appBundleId, - fullscreen: screenshotOptions.fullscreen, - stabilize: screenshotOptions.stabilize, - surface: context?.surface, - }); - return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; - } - case 'back': - await interactor.back(context?.backMode); - return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; - case 'home': - await interactor.home(); - return { action: 'home', ...successText('Home') }; - case 'rotate': { - const orientation = parseDeviceRotation(positionals[0]); - await interactor.rotate(orientation); - return { - action: 'rotate', - orientation, - ...successText(`Rotated to ${orientation}`), - }; - } - case 'app-switcher': - await interactor.appSwitcher(); - return { action: 'app-switcher', ...successText('Opened app switcher') }; - case 'clipboard': - return handleClipboardCommand(interactor, positionals); - case 'keyboard': - return handleKeyboardCommand(device, positionals, context, runnerCtx); - case 'settings': - return handleSettingsCommand(device, interactor, positionals, context); - case 'push': - return handlePushCommand(device, positionals, context); - case 'snapshot': - return await handleSnapshotCommand(interactor, context); - case 'read': - return handleReadCommand(device, positionals, context); - default: - throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); - } + return await dispatchKnownCommand( + device, + interactor, + command, + positionals, + outPath, + context, + runnerCtx, + ); }, { command, @@ -171,6 +89,84 @@ export async function dispatchCommand( ); } +// fallow-ignore-next-line complexity +async function dispatchKnownCommand( + device: DeviceInfo, + interactor: Interactor, + command: string, + positionals: string[], + outPath: string | undefined, + context: DispatchContext | undefined, + runnerCtx: RunnerContext, +): Promise | void> { + switch (command) { + case 'open': + return await handleOpenCommand(device, interactor, positionals, context); + case 'close': { + const app = positionals[0]; + if (!app) return { closed: 'session', ...successText('Closed session') }; + await interactor.close(app); + return { app, ...successText(`Closed: ${app}`) }; + } + case 'press': + return await handlePressCommand(device, interactor, positionals, context); + case 'swipe': + return await handleSwipeCommand(device, interactor, positionals, context); + case 'pan': + return await handlePanCommand(interactor, positionals); + case 'fling': + return await handleFlingCommand(interactor, positionals); + case 'longpress': + return await handleLongPressCommand(interactor, positionals); + case 'focus': + return await handleFocusCommand(interactor, positionals); + case 'type': + return await handleTypeCommand(interactor, positionals, context); + case 'fill': + return await handleFillCommand(interactor, positionals, context); + case 'scroll': + return await handleScrollCommand(interactor, positionals, context); + case 'pinch': + return await handlePinchCommand(device, interactor, positionals, context); + case 'rotate-gesture': + return await handleRotateGestureCommand(device, interactor, positionals); + case 'transform-gesture': + return await handleTransformGestureCommand(device, interactor, positionals); + case 'trigger-app-event': + return await handleTriggerAppEventCommand(device, interactor, positionals, context); + case 'screenshot': + return await handleScreenshotCommand(interactor, positionals, outPath, context); + case 'back': + await interactor.back(context?.backMode); + return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') }; + case 'home': + await interactor.home(); + return { action: 'home', ...successText('Home') }; + case 'rotate': { + const orientation = parseDeviceRotation(positionals[0]); + await interactor.rotate(orientation); + return { action: 'rotate', orientation, ...successText(`Rotated to ${orientation}`) }; + } + case 'app-switcher': + await interactor.appSwitcher(); + return { action: 'app-switcher', ...successText('Opened app switcher') }; + case 'clipboard': + return await handleClipboardCommand(interactor, positionals); + case 'keyboard': + return await handleKeyboardCommand(device, positionals, context, runnerCtx); + case 'settings': + return await handleSettingsCommand(device, interactor, positionals, context); + case 'push': + return await handlePushCommand(device, positionals, context); + case 'snapshot': + return await handleSnapshotCommand(interactor, context); + case 'read': + return await handleReadCommand(device, positionals, context); + default: + throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); + } +} + // --------------------------------------------------------------------------- // Command handlers // --------------------------------------------------------------------------- @@ -199,9 +195,6 @@ async function handleOpenCommand( throw new AppError('UNSUPPORTED_OPERATION', LAUNCH_CONSOLE_IOS_SIMULATOR_ONLY_MESSAGE); } if (url !== undefined) { - if (device.platform === 'android') { - throw new AppError('INVALID_ARGS', 'open is supported only on Apple platforms'); - } if (isDeepLinkTarget(app)) { throw new AppError( 'INVALID_ARGS', @@ -217,6 +210,7 @@ async function handleOpenCommand( await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, + launchArgs: context?.launchArgs, url, }); return { app, url, ...successText(`Opened: ${app}`) }; @@ -224,14 +218,72 @@ async function handleOpenCommand( if (launchConsole && isDeepLinkTarget(app)) { throw new AppError('INVALID_ARGS', LAUNCH_CONSOLE_DIRECT_APP_ONLY_MESSAGE); } + if (device.platform === 'android' && context?.launchArgs && context.launchArgs.length > 0) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Launch arguments are currently supported only on Apple platforms.', + ); + } + if (context?.clearAppState) { + if (isDeepLinkTarget(app)) { + throw new AppError( + 'INVALID_ARGS', + 'Clearing app state requires an app target, not a deep link.', + ); + } + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Clearing app state is currently supported only on iOS simulators.', + ); + } + await clearIosSimulatorAppState(device, app); + } await interactor.open(app, { activity: context?.activity, appBundleId: context?.appBundleId, launchConsole, + launchArgs: context?.launchArgs, }); return { app, ...(launchConsole ? { launchConsole } : {}), ...successText(`Opened: ${app}`) }; } +async function handleTriggerAppEventCommand( + device: DeviceInfo, + interactor: Interactor, + positionals: string[], + context: DispatchContext | undefined, +): Promise> { + const { eventName, payload } = parseTriggerAppEventArgs(positionals); + const eventUrl = resolveAppEventUrl(device.platform, eventName, payload); + await interactor.open(eventUrl, { appBundleId: context?.appBundleId }); + return { + event: eventName, + eventUrl, + transport: 'deep-link', + ...successText(`Triggered app event: ${eventName}`), + }; +} + +async function handleScreenshotCommand( + interactor: Interactor, + positionals: string[], + outPath: string | undefined, + context: DispatchContext | undefined, +): Promise> { + const positionalPath = positionals[0]; + const screenshotPath = positionalPath ?? outPath ?? `./screenshot-${Date.now()}.png`; + await fs.mkdir(pathModule.dirname(screenshotPath), { recursive: true }); + const screenshotOptions = screenshotOptionsFromFlags(context); + await interactor.screenshot(screenshotPath, { + appBundleId: context?.appBundleId, + fullscreen: screenshotOptions.fullscreen, + stabilize: screenshotOptions.stabilize, + surface: context?.surface, + }); + return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) }; +} + async function handleClipboardCommand( interactor: Interactor, positionals: string[], @@ -266,65 +318,106 @@ async function handleKeyboardCommand( runnerCtx: RunnerContext, ): Promise> { const action = (positionals[0] ?? 'status').toLowerCase(); - if (action !== 'status' && action !== 'get' && action !== 'dismiss') { - throw new AppError('INVALID_ARGS', 'keyboard requires a subcommand: status, get, or dismiss'); + if (!isKeyboardAction(action)) { + throw new AppError( + 'INVALID_ARGS', + 'keyboard requires a subcommand: status, get, dismiss, enter, or return', + ); } if (positionals.length > 1) { throw new AppError('INVALID_ARGS', 'keyboard accepts at most one subcommand argument'); } if (device.platform === 'android') { - if (action === 'dismiss') { - const result = await dismissAndroidKeyboard(device); - return { - platform: 'android', - action: 'dismiss', - attempts: result.attempts, - wasVisible: result.wasVisible, - dismissed: result.dismissed, - visible: result.visible, - inputType: result.inputType, - type: result.type, - inputMethodPackage: result.inputMethodPackage, - focusedPackage: result.focusedPackage, - focusedResourceId: result.focusedResourceId, - inputOwner: result.inputOwner, - }; - } - const state = await getAndroidKeyboardState(device); + return await handleAndroidKeyboardCommand(device, action); + } + if (device.platform === 'ios') { + return await handleIosKeyboardCommand(device, action, context, runnerCtx); + } + throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); +} + +async function handleAndroidKeyboardCommand( + device: DeviceInfo, + action: KeyboardAction, +): Promise> { + if (action === 'enter' || action === 'return') { + await pressAndroidEnter(device); return { platform: 'android', - action: 'status', - visible: state.visible, - inputType: state.inputType, - type: state.type, - inputMethodPackage: state.inputMethodPackage, - focusedPackage: state.focusedPackage, - focusedResourceId: state.focusedResourceId, - inputOwner: state.inputOwner, + action: 'enter', + ...successText('Keyboard enter pressed'), }; } - if (device.platform === 'ios') { - if (action !== 'dismiss') { - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'keyboard status/get is currently supported only on Android; use keyboard dismiss on iOS', - ); - } + if (action === 'dismiss') { + const result = await dismissAndroidKeyboard(device); + return { + platform: 'android', + action: 'dismiss', + attempts: result.attempts, + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + inputType: result.inputType, + type: result.type, + inputMethodPackage: result.inputMethodPackage, + focusedPackage: result.focusedPackage, + focusedResourceId: result.focusedResourceId, + inputOwner: result.inputOwner, + }; + } + const state = await getAndroidKeyboardState(device); + return { + platform: 'android', + action: 'status', + visible: state.visible, + inputType: state.inputType, + type: state.type, + inputMethodPackage: state.inputMethodPackage, + focusedPackage: state.focusedPackage, + focusedResourceId: state.focusedResourceId, + inputOwner: state.inputOwner, + }; +} + +async function handleIosKeyboardCommand( + device: DeviceInfo, + action: KeyboardAction, + context: DispatchContext | undefined, + runnerCtx: RunnerContext, +): Promise> { + if (action !== 'dismiss' && action !== 'enter' && action !== 'return') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'keyboard status/get is currently supported only on Android; use keyboard dismiss or enter on iOS', + ); + } + if (action === 'enter' || action === 'return') { const result = await runIosRunnerCommand( device, - { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + { command: 'keyboardReturn', appBundleId: context?.appBundleId }, runnerCtx, ); return { platform: 'ios', - action: 'dismiss', - wasVisible: result.wasVisible, - dismissed: result.dismissed, + action: 'enter', visible: result.visible, - ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + wasVisible: result.wasVisible, + ...successText('Keyboard enter pressed'), }; } - throw new AppError('UNSUPPORTED_OPERATION', 'keyboard is supported only on Android and iOS'); + const result = await runIosRunnerCommand( + device, + { command: 'keyboardDismiss', appBundleId: context?.appBundleId }, + runnerCtx, + ); + return { + platform: 'ios', + action: 'dismiss', + wasVisible: result.wasVisible, + dismissed: result.dismissed, + visible: result.visible, + ...successText(result.dismissed ? 'Keyboard dismissed' : 'Keyboard already hidden'), + }; } async function handleSettingsCommand( diff --git a/src/core/interactor-types.ts b/src/core/interactor-types.ts index 01653a5e9..2ddf377a8 100644 --- a/src/core/interactor-types.ts +++ b/src/core/interactor-types.ts @@ -29,6 +29,7 @@ export type ScreenshotOptions = { export type ElementSelectorTapOptions = { key: 'id' | 'label' | 'text' | 'value'; value: string; + allowNonHittableCoordinateFallback?: boolean; }; export type SnapshotOptions = BaseSnapshotOptions & { @@ -44,7 +45,13 @@ export type SnapshotResult = Omit & export type Interactor = { open( app: string, - options?: { activity?: string; appBundleId?: string; launchConsole?: string; url?: string }, + options?: { + activity?: string; + appBundleId?: string; + launchConsole?: string; + launchArgs?: string[]; + url?: string; + }, ): Promise; openDevice(): Promise; close(app: string): Promise; @@ -75,6 +82,11 @@ export type Interactor = { longPress(x: number, y: number, durationMs?: number): Promise | void>; focus(x: number, y: number): Promise | void>; type(text: string, delayMs?: number): Promise; + fillElementSelector?( + selector: ElementSelectorTapOptions, + text: string, + delayMs?: number, + ): Promise | void>; fill( x: number, y: number, diff --git a/src/core/interactors/android.ts b/src/core/interactors/android.ts index 3dc7f2993..8ebb77220 100644 --- a/src/core/interactors/android.ts +++ b/src/core/interactors/android.ts @@ -34,7 +34,12 @@ import type { Interactor } from '../interactor-types.ts'; export function createAndroidInteractor(device: DeviceInfo): Interactor { return { - open: (app, options) => openAndroidApp(device, app, options?.activity), + open: (app, options) => + openAndroidApp(device, app, { + activity: options?.activity, + appBundleId: options?.appBundleId, + url: options?.url, + }), openDevice: () => openAndroidDevice(device), close: (app) => closeAndroidApp(device, app), tap: (x, y) => pressAndroid(device, x, y), diff --git a/src/core/interactors/apple.ts b/src/core/interactors/apple.ts index a841b8c99..744adcb01 100644 --- a/src/core/interactors/apple.ts +++ b/src/core/interactors/apple.ts @@ -30,6 +30,7 @@ export function createAppleInteractor( openIosApp(device, app, { appBundleId: options?.appBundleId, launchConsole: options?.launchConsole, + launchArgs: options?.launchArgs, url: options?.url, }), openDevice: () => openIosDevice(device), diff --git a/src/daemon-client.ts b/src/daemon-client.ts index ba17f166e..4dce1e4fc 100644 --- a/src/daemon-client.ts +++ b/src/daemon-client.ts @@ -2,6 +2,7 @@ import net from 'node:net'; import http from 'node:http'; import https from 'node:https'; import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { pipeline } from 'node:stream/promises'; import { sleep } from './utils/timeouts.ts'; @@ -26,6 +27,7 @@ import { } from './daemon/config.ts'; import { uploadArtifact } from './upload-client.ts'; import { computeDaemonCodeSignature } from './daemon/code-signature.ts'; +import { PUBLIC_COMMANDS } from './command-catalog.ts'; export { computeDaemonCodeSignature } from './daemon/code-signature.ts'; export type DaemonRequest = SharedDaemonRequest; export type DaemonResponse = SharedDaemonResponse; @@ -90,10 +92,16 @@ type DaemonClientSettings = { paths: DaemonPaths; transportPreference: DaemonTransportPreference; serverMode: DaemonServerMode; + ownedStateDir?: boolean; remoteBaseUrl?: string; remoteAuthToken?: string; }; +type EnsuredDaemon = { + info: DaemonInfo; + startedByClient: boolean; +}; + type ResolvedDaemonTransport = 'socket' | 'http'; const REQUEST_TIMEOUT_MS = 90_000; @@ -117,12 +125,13 @@ export async function sendToDaemon(req: Omit): Promise await ensureDaemon(settings), { requestId, session: req.session }, ); + const info = daemon.info; const preparedRemoteRequest = await prepareRemoteRequest(req, info); const request = { @@ -161,11 +170,23 @@ export async function sendToDaemon(req: Omit): Promise await sendRequest(info, request, settings.transportPreference, requestTimeoutMs), - { requestId, command: req.command }, - ); + try { + return await withDiagnosticTimer( + 'daemon_request', + async () => await sendRequest(info, request, settings.transportPreference, requestTimeoutMs), + { requestId, command: req.command }, + ); + } finally { + await cleanupDaemonAfterRequest(req, daemon, settings); + } +} + +function resolveDaemonRequestTimeoutMs(req: Omit): number | undefined { + if (req.command === PUBLIC_COMMANDS.test) return undefined; + if (req.command === PUBLIC_COMMANDS.replay && typeof req.flags?.timeoutMs === 'number') { + return req.flags.timeoutMs; + } + return REQUEST_TIMEOUT_MS; } export async function openApp(options: OpenAppOptions = {}): Promise { @@ -227,18 +248,7 @@ async function prepareRemoteRequest( let uploadedArtifactId: string | undefined; if (isRemoteDaemon(info)) { - const remoteArtifact = prepareRemoteArtifactCommand(req, positionals); - if (remoteArtifact) { - if (remoteArtifact.positionalPath !== undefined) { - positionals[remoteArtifact.positionalIndex] = remoteArtifact.positionalPath; - } - if (remoteArtifact.flagPath !== undefined) { - flags ??= {}; - flags.out = remoteArtifact.flagPath; - } - clientArtifactPaths[remoteArtifact.field] = remoteArtifact.localPath; - } - + flags = applyRemoteArtifactCommand(req, positionals, flags, clientArtifactPaths); const remoteInstallSource = await prepareRemoteInstallSource(req, info); if (remoteInstallSource) { installSource = remoteInstallSource.installSource; @@ -269,10 +279,8 @@ async function prepareRemoteRequest( return createPreparedRemoteRequest({ positionals, flags, clientArtifactPaths }); } - const localPath = path.isAbsolute(rawPath) - ? rawPath - : path.resolve(req.meta?.cwd ?? process.cwd(), rawPath); - if (!fs.existsSync(localPath)) { + const localPath = resolveLocalInstallPath(rawPath, req.meta?.cwd); + if (!localPath) { return createPreparedRemoteRequest({ positionals, flags, clientArtifactPaths }); } @@ -285,6 +293,37 @@ async function prepareRemoteRequest( return baseResult(); } +function applyRemoteArtifactCommand( + req: Omit, + positionals: string[], + flags: DaemonRequest['flags'] | undefined, + clientArtifactPaths: Record, +): DaemonRequest['flags'] | undefined { + const remoteArtifact = prepareRemoteArtifactCommand(req, positionals); + if (!remoteArtifact) return flags; + if (remoteArtifact.positionalPath !== undefined) { + positionals[remoteArtifact.positionalIndex] = remoteArtifact.positionalPath; + } + const nextFlags = applyRemoteArtifactOutFlag(flags, remoteArtifact.flagPath); + clientArtifactPaths[remoteArtifact.field] = remoteArtifact.localPath; + return nextFlags; +} + +function applyRemoteArtifactOutFlag( + flags: DaemonRequest['flags'] | undefined, + flagPath: string | undefined, +): DaemonRequest['flags'] | undefined { + if (flagPath === undefined) return flags; + return { ...(flags ?? {}), out: flagPath }; +} + +function resolveLocalInstallPath(rawPath: string, cwd: string | undefined): string | undefined { + const localPath = path.isAbsolute(rawPath) + ? rawPath + : path.resolve(cwd ?? process.cwd(), rawPath); + return fs.existsSync(localPath) ? localPath : undefined; +} + type PreparedRemoteRequest = { positionals: string[]; flags?: DaemonRequest['flags']; @@ -419,10 +458,11 @@ function buildRemoteTempArtifactPath(prefix: string, extension: string): string } function resolveClientSettings(req: Omit): DaemonClientSettings { - const stateDir = req.flags?.stateDir ?? process.env.AGENT_DEVICE_STATE_DIR; - const remoteBaseUrl = resolveRemoteDaemonBaseUrl( - req.flags?.daemonBaseUrl ?? process.env.AGENT_DEVICE_DAEMON_BASE_URL, - ); + const explicitStateDir = req.flags?.stateDir ?? process.env.AGENT_DEVICE_STATE_DIR; + const rawRemoteBaseUrl = req.flags?.daemonBaseUrl ?? process.env.AGENT_DEVICE_DAEMON_BASE_URL; + const useOwnedReplayStateDir = + isOneShotReplayCommand(req.command) && !explicitStateDir && !rawRemoteBaseUrl; + const remoteBaseUrl = resolveRemoteDaemonBaseUrl(rawRemoteBaseUrl); const remoteAuthToken = req.flags?.daemonAuthToken ?? process.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; validateRemoteDaemonTrust(remoteBaseUrl, remoteAuthToken); const rawTransport = req.flags?.daemonTransport ?? process.env.AGENT_DEVICE_DAEMON_TRANSPORT; @@ -439,57 +479,69 @@ function resolveClientSettings(req: Omit): DaemonClientS process.env.AGENT_DEVICE_DAEMON_SERVER_MODE ?? (rawTransport === 'dual' ? 'dual' : undefined); const serverMode = resolveDaemonServerMode(rawServerMode); + const stateDir = useOwnedReplayStateDir + ? fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-replay-daemon-')) + : explicitStateDir; return { paths: resolveDaemonPaths(stateDir), transportPreference, serverMode, + ownedStateDir: useOwnedReplayStateDir, remoteBaseUrl, remoteAuthToken, }; } -async function ensureDaemon(settings: DaemonClientSettings): Promise { +async function ensureDaemon(settings: DaemonClientSettings): Promise { if (settings.remoteBaseUrl) { - const remoteInfo: DaemonInfo = { - transport: 'http', - // Remote mode reuses the auth token as the daemon token so the existing JSON-RPC contract still works. - token: settings.remoteAuthToken ?? '', - pid: 0, - baseUrl: settings.remoteBaseUrl, - }; - if (await canConnect(remoteInfo, 'http')) return remoteInfo; - throw new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { - daemonBaseUrl: settings.remoteBaseUrl, - hint: 'Verify AGENT_DEVICE_DAEMON_BASE_URL points to a reachable daemon with GET /health and POST /rpc.', - }); + return await ensureRemoteDaemon(settings); } - const existing = readDaemonInfo(settings.paths.infoPath); - const localVersion = readVersion(); - const localCodeSignature = resolveLocalDaemonCodeSignature(); - const existingReachable = existing - ? await canConnect(existing, settings.transportPreference) - : false; - if ( - existing && - existing.version === localVersion && - existing.codeSignature === localCodeSignature && - existingReachable - ) { - return existing; - } - if ( - existing && - (existing.version !== localVersion || - existing.codeSignature !== localCodeSignature || - !existingReachable) - ) { - await stopDaemonProcessForTakeover(existing); - removeDaemonInfo(settings.paths.infoPath); - } + const reusable = await readReusableLocalDaemon(settings); + if (reusable) return { info: reusable, startedByClient: false }; cleanupStaleDaemonLockIfSafe(settings.paths); + return await startLocalDaemon(settings); +} + +async function ensureRemoteDaemon(settings: DaemonClientSettings): Promise { + const remoteInfo: DaemonInfo = { + transport: 'http', + // Remote mode reuses the auth token as the daemon token so the existing JSON-RPC contract still works. + token: settings.remoteAuthToken ?? '', + pid: 0, + baseUrl: settings.remoteBaseUrl, + }; + if (await canConnect(remoteInfo, 'http')) { + return { info: remoteInfo, startedByClient: false }; + } + throw new AppError('COMMAND_FAILED', 'Remote daemon is unavailable', { + daemonBaseUrl: settings.remoteBaseUrl, + hint: 'Verify AGENT_DEVICE_DAEMON_BASE_URL points to a reachable daemon with GET /health and POST /rpc.', + }); +} + +async function readReusableLocalDaemon(settings: DaemonClientSettings): Promise { + const existing = readDaemonInfo(settings.paths.infoPath); + if (!existing) return null; + + const existingReachable = await canConnect(existing, settings.transportPreference); + if (isReusableDaemonInfo(existing, existingReachable)) return existing; + + await stopDaemonProcessForTakeover(existing); + removeDaemonInfo(settings.paths.infoPath); + return null; +} + +function isReusableDaemonInfo(info: DaemonInfo, reachable: boolean): boolean { + return ( + info.version === readVersion() && + info.codeSignature === resolveLocalDaemonCodeSignature() && + reachable + ); +} +async function startLocalDaemon(settings: DaemonClientSettings): Promise { let lockRecoveryCount = 0; const cleanupResults: DaemonStartupCleanupResult[] = []; let startError: string | undefined; @@ -507,7 +559,7 @@ async function ensureDaemon(settings: DaemonClientSettings): Promise } const started = await waitForDaemonInfo(DAEMON_STARTUP_TIMEOUT_MS, settings); - if (started) return started; + if (started) return { info: started, startedByClient: true }; if (await recoverDaemonLockHolder(settings.paths)) { lockRecoveryCount += 1; @@ -522,7 +574,7 @@ async function ensureDaemon(settings: DaemonClientSettings): Promise cleanupResults.push(cleanup); if (cleanup.retainedInfoProcess || cleanup.retainedLockProcess) { const extended = await waitForDaemonInfo(DAEMON_STARTUP_TIMEOUT_MS, settings); - if (extended) return extended; + if (extended) return { info: extended, startedByClient: true }; break; } if (!hasAnotherAttempt) break; @@ -546,6 +598,57 @@ async function ensureDaemon(settings: DaemonClientSettings): Promise }); } +async function cleanupDaemonAfterRequest( + req: Omit, + daemon: EnsuredDaemon, + settings: DaemonClientSettings, +): Promise { + if ( + !isOneShotReplayCommand(req.command) || + (!daemon.startedByClient && !settings.ownedStateDir) || + isRemoteDaemon(daemon.info) + ) { + return; + } + + const result = { + pid: daemon.info.pid, + removedInfo: false, + removedLock: false, + removedStateDir: false, + error: undefined as string | undefined, + }; + + try { + await stopDaemonProcessForTakeover(daemon.info); + } catch (error) { + result.error = error instanceof Error ? error.message : String(error); + } finally { + const infoExists = fs.existsSync(settings.paths.infoPath); + removeDaemonInfo(settings.paths.infoPath); + result.removedInfo = infoExists && !fs.existsSync(settings.paths.infoPath); + + const lockExists = fs.existsSync(settings.paths.lockPath); + removeDaemonLock(settings.paths.lockPath); + result.removedLock = lockExists && !fs.existsSync(settings.paths.lockPath); + + if (settings.ownedStateDir) { + fs.rmSync(settings.paths.baseDir, { recursive: true, force: true }); + result.removedStateDir = !fs.existsSync(settings.paths.baseDir); + } + } + + emitDiagnostic({ + level: result.error ? 'warn' : 'info', + phase: 'daemon_replay_cleanup', + data: result, + }); +} + +function isOneShotReplayCommand(command: string | undefined): boolean { + return command === PUBLIC_COMMANDS.replay || command === PUBLIC_COMMANDS.test; +} + async function waitForDaemonInfo( timeoutMs: number, settings: DaemonClientSettings, @@ -586,32 +689,46 @@ function readDaemonInfo(infoPath: string): DaemonInfo | null { const data = readJsonFile(infoPath); if (!data || typeof data !== 'object') return null; const parsed = data as Partial; - const token = typeof parsed.token === 'string' && parsed.token.length > 0 ? parsed.token : null; + const token = readRequiredDaemonToken(parsed); if (!token) return null; - const hasSocket = Number.isInteger(parsed.port) && Number(parsed.port) > 0; - const hasHttp = Number.isInteger(parsed.httpPort) && Number(parsed.httpPort) > 0; - if (!hasSocket && !hasHttp) return null; - const transport = parsed.transport; - const version = typeof parsed.version === 'string' ? parsed.version : undefined; - const codeSignature = typeof parsed.codeSignature === 'string' ? parsed.codeSignature : undefined; - const processStartTime = - typeof parsed.processStartTime === 'string' ? parsed.processStartTime : undefined; - const hasPid = Number.isInteger(parsed.pid) && Number(parsed.pid) > 0; + const ports = readDaemonInfoPorts(parsed); + if (!ports) return null; return { token, - port: hasSocket ? Number(parsed.port) : undefined, - httpPort: hasHttp ? Number(parsed.httpPort) : undefined, - transport: - transport === 'socket' || transport === 'http' || transport === 'dual' - ? transport - : undefined, - pid: hasPid ? Number(parsed.pid) : 0, - version, - codeSignature, - processStartTime, + ...ports, + transport: readDaemonInfoTransport(parsed.transport), + pid: readPositiveInteger(parsed.pid) ?? 0, + version: readOptionalString(parsed.version), + codeSignature: readOptionalString(parsed.codeSignature), + processStartTime: readOptionalString(parsed.processStartTime), }; } +function readRequiredDaemonToken(parsed: Partial): string | null { + return typeof parsed.token === 'string' && parsed.token.length > 0 ? parsed.token : null; +} + +function readDaemonInfoPorts( + parsed: Partial, +): Pick | null { + const port = readPositiveInteger(parsed.port); + const httpPort = readPositiveInteger(parsed.httpPort); + if (port === undefined && httpPort === undefined) return null; + return { port, httpPort }; +} + +function readDaemonInfoTransport(value: unknown): DaemonInfo['transport'] { + return value === 'socket' || value === 'http' || value === 'dual' ? value : undefined; +} + +function readOptionalString(value: unknown): string | undefined { + return typeof value === 'string' ? value : undefined; +} + +function readPositiveInteger(value: unknown): number | undefined { + return Number.isInteger(value) && Number(value) > 0 ? Number(value) : undefined; +} + function readDaemonLockInfo(lockPath: string): DaemonLockInfo | null { const data = readJsonFile(lockPath); if (!data || typeof data !== 'object') return null; @@ -753,15 +870,25 @@ async function canConnect( return await canConnectSocket(info.port); } -function canConnectSocket(port: number | undefined): Promise { +export function canConnectSocket(port: number | undefined): Promise { if (!port) return Promise.resolve(false); return new Promise((resolve) => { + let settled = false; const socket = net.createConnection({ host: '127.0.0.1', port }, () => { + finish(true); + }); + const finish = (reachable: boolean) => { + if (settled) return; + settled = true; socket.destroy(); - resolve(true); + resolve(reachable); + }; + socket.setTimeout(LOCAL_DAEMON_HEALTHCHECK_TIMEOUT_MS); + socket.on('timeout', () => { + finish(false); }); socket.on('error', () => { - resolve(false); + finish(false); }); }); } @@ -1150,28 +1277,9 @@ function handleDaemonHttpResponseBody( ): void { const { info, req, resolve, reject } = options; try { - const parsed = JSON.parse(body) as { - result?: DaemonResponse; - error?: { - message?: string; - data?: Record; - }; - }; + const parsed = parseDaemonHttpResponseBody(body); if (parsed.error) { - const data = parsed.error.data ?? {}; - reject( - new AppError( - toAppErrorCode(data.code != null ? String(data.code) : undefined, 'COMMAND_FAILED'), - String(data.message ?? parsed.error.message ?? 'Daemon RPC request failed'), - { - ...(typeof data.details === 'object' && data.details ? data.details : {}), - hint: typeof data.hint === 'string' ? data.hint : undefined, - diagnosticId: typeof data.diagnosticId === 'string' ? data.diagnosticId : undefined, - logPath: typeof data.logPath === 'string' ? data.logPath : undefined, - requestId: req.meta?.requestId, - }, - ), - ); + reject(toDaemonHttpRpcError(parsed.error, req.meta?.requestId)); return; } if (!parsed.result || typeof parsed.result !== 'object') { @@ -1182,11 +1290,7 @@ function handleDaemonHttpResponseBody( ); return; } - if (info.baseUrl && parsed.result.ok) { - void materializeRemoteArtifacts(info, req, parsed.result).then(resolve).catch(reject); - return; - } - resolve(parsed.result); + void resolveDaemonHttpResult(info, req, parsed.result, resolve, reject); } catch (err) { reject( new AppError( @@ -1202,6 +1306,50 @@ function handleDaemonHttpResponseBody( } } +function parseDaemonHttpResponseBody(body: string): { + result?: DaemonResponse; + error?: { message?: string; data?: Record }; +} { + return JSON.parse(body) as { + result?: DaemonResponse; + error?: { message?: string; data?: Record }; + }; +} + +function toDaemonHttpRpcError( + error: { message?: string; data?: Record }, + requestId: string | undefined, +): AppError { + const data = error.data ?? {}; + return new AppError( + toAppErrorCode(data.code != null ? String(data.code) : undefined, 'COMMAND_FAILED'), + String(data.message ?? error.message ?? 'Daemon RPC request failed'), + { + ...(typeof data.details === 'object' && data.details ? data.details : {}), + hint: typeof data.hint === 'string' ? data.hint : undefined, + diagnosticId: typeof data.diagnosticId === 'string' ? data.diagnosticId : undefined, + logPath: typeof data.logPath === 'string' ? data.logPath : undefined, + requestId, + }, + ); +} + +async function resolveDaemonHttpResult( + info: DaemonInfo, + req: DaemonRequest, + result: DaemonResponse, + resolve: (response: DaemonResponse | PromiseLike) => void, + reject: (error: unknown) => void, +): Promise { + try { + resolve( + info.baseUrl && result.ok ? await materializeRemoteArtifacts(info, req, result) : result, + ); + } catch (error) { + reject(error); + } +} + function buildHttpRpcPayload( req: DaemonRequest, options: { includeTokenParam: boolean }, @@ -1498,13 +1646,13 @@ export async function downloadRemoteArtifact(params: { }, ); const timeoutHandle = setTimeout(() => { - request.destroy( - new AppError('COMMAND_FAILED', 'Remote artifact download timed out', { - artifactId: params.artifactId, - requestId: params.requestId, - timeoutMs, - }), - ); + const timeoutError = new AppError('COMMAND_FAILED', 'Remote artifact download timed out', { + artifactId: params.artifactId, + requestId: params.requestId, + timeoutMs, + }); + settle(timeoutError); + request.destroy(timeoutError); }, timeoutMs); request.on('error', (error) => { if (error instanceof AppError) { diff --git a/src/daemon/__tests__/context.test.ts b/src/daemon/__tests__/context.test.ts index 090314199..92c1b6339 100644 --- a/src/daemon/__tests__/context.test.ts +++ b/src/daemon/__tests__/context.test.ts @@ -14,6 +14,12 @@ test('contextFromFlags forwards scroll pixels from CLI flags', () => { assert.equal(context.pixels, 240); }); +test('contextFromFlags forwards generic app-state clearing', () => { + const flags: CommandFlags = { clearAppState: true }; + const context = contextFromFlags('/tmp/agent-device.log', flags); + assert.equal(context.clearAppState, true); +}); + test('contextFromFlags forwards screenshot flags from CLI flags', () => { const flags: CommandFlags = { screenshotFullscreen: true, diff --git a/src/daemon/__tests__/post-gesture-stabilization.test.ts b/src/daemon/__tests__/post-gesture-stabilization.test.ts index a96b4a498..77270fb78 100644 --- a/src/daemon/__tests__/post-gesture-stabilization.test.ts +++ b/src/daemon/__tests__/post-gesture-stabilization.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { afterEach, test, vi } from 'vitest'; -import { IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; +import { ANDROID_EMULATOR, IOS_SIMULATOR } from '../../__tests__/test-utils/device-fixtures.ts'; import type { SnapshotState } from '../../utils/snapshot.ts'; import { capturePostGestureStabilizedSnapshot, @@ -12,6 +12,22 @@ afterEach(() => { vi.useRealTimers(); }); +test('markPostGestureStabilization marks iOS swipe sessions', () => { + const session = makeSession(); + + markPostGestureStabilization(session, 'swipe'); + + assert.equal(session.postGestureStabilization?.action, 'swipe'); +}); + +test('markPostGestureStabilization marks Android swipe sessions', () => { + const session = makeSession('android'); + + markPostGestureStabilization(session, 'swipe'); + + assert.equal(session.postGestureStabilization?.action, 'swipe'); +}); + test('capturePostGestureStabilizedSnapshot retries until rects stop moving', async () => { vi.useFakeTimers(); const session = makeSession(); @@ -30,10 +46,10 @@ test('capturePostGestureStabilizedSnapshot retries until rects stop moving', asy assert.equal(session.postGestureStabilization, undefined); }); -function makeSession(): SessionState { +function makeSession(platform: 'ios' | 'android' = 'ios'): SessionState { return { - name: 'ios', - device: IOS_SIMULATOR, + name: platform, + device: platform === 'android' ? ANDROID_EMULATOR : IOS_SIMULATOR, createdAt: Date.now(), actions: [], }; diff --git a/src/daemon/__tests__/request-lock-policy.test.ts b/src/daemon/__tests__/request-lock-policy.test.ts index 03e5d6f41..d6b54505d 100644 --- a/src/daemon/__tests__/request-lock-policy.test.ts +++ b/src/daemon/__tests__/request-lock-policy.test.ts @@ -291,4 +291,3 @@ test('strips only conflicting selectors for existing sessions', () => { assert.equal(req.flags?.device, 'iPhone 16'); assert.equal(req.flags?.serial, undefined); }); - diff --git a/src/daemon/__tests__/selectors.test.ts b/src/daemon/__tests__/selectors.test.ts index 1d42dbdb1..2849f19df 100644 --- a/src/daemon/__tests__/selectors.test.ts +++ b/src/daemon/__tests__/selectors.test.ts @@ -290,4 +290,3 @@ test('appName selector matches nodes with appName field', () => { assert.ok(match3); assert.equal(match3.matches, 1); }); - diff --git a/src/daemon/__tests__/session-routing.test.ts b/src/daemon/__tests__/session-routing.test.ts index 7bb5daea0..660d6dfdf 100644 --- a/src/daemon/__tests__/session-routing.test.ts +++ b/src/daemon/__tests__/session-routing.test.ts @@ -47,4 +47,3 @@ test('reuses lone active session for implicit default session', (t) => { assert.equal(resolved, 'android'); }); - diff --git a/src/daemon/context.ts b/src/daemon/context.ts index 05c0864d4..7947241d0 100644 --- a/src/daemon/context.ts +++ b/src/daemon/context.ts @@ -8,6 +8,9 @@ import { getDiagnosticsMeta } from '../utils/diagnostics.ts'; export type DaemonCommandContext = DispatchContext & ScreenshotRuntimeFlags; +// Flat compatibility mapper: keeping each CLI flag visible here makes request +// context drift easier to spot than splitting the same optional fields apart. +// fallow-ignore-next-line complexity export function contextFromFlags( logPath: string, flags: CommandFlags | undefined, @@ -21,6 +24,8 @@ export function contextFromFlags( appBundleId, activity: flags?.activity, launchConsole: flags?.launchConsole, + launchArgs: flags?.launchArgs, + clearAppState: flags?.clearAppState, verbose: flags?.verbose, logPath, traceLogPath, diff --git a/src/daemon/direct-ios-selector.ts b/src/daemon/direct-ios-selector.ts index 5063f6632..60dfe075f 100644 --- a/src/daemon/direct-ios-selector.ts +++ b/src/daemon/direct-ios-selector.ts @@ -6,6 +6,7 @@ export type DirectIosSelectorTarget = { key: 'id' | 'label' | 'text' | 'value'; value: string; raw: string; + allowNonHittableCoordinateFallback?: boolean; }; export function readSimpleIosSelectorTarget(params: { diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 7efbca153..8c9b92c0d 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -95,6 +95,7 @@ test('handleFindCommands click returns deterministic metadata across locator var expectedLocator: 'any', expectedQuery: 'Increment', expectedCoordinates: { x: 100, y: 50 }, + expectedRef: '@e2', }, ]; @@ -104,7 +105,7 @@ test('handleFindCommands click returns deterministic metadata across locator var if (!response.ok) return; const data = response.data as Record; expect(Object.keys(data).sort()).toEqual(scenario.expectedKeys); - expect(data.ref).toBe('@e1'); + expect(data.ref).toBe(scenario.expectedRef); expect(data.locator).toBe(scenario.expectedLocator); expect(data.query).toBe(scenario.expectedQuery); @@ -117,10 +118,101 @@ test('handleFindCommands click returns deterministic metadata across locator var } expect(invokeCalls.length).toBe(1); - expect(invokeCalls[0].positionals?.[0]).toBe('@e1'); + expect(invokeCalls[0].positionals?.[0]).toBe(scenario.expectedRef); } }); +test('handleFindCommands click prefers on-screen duplicate text matches', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Sign in', 'click'], + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: -199, y: 186, width: 70, height: 33 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'Button', + label: 'Sign in', + hittable: false, + rect: { x: 40, y: 870, width: 360, height: 44 }, + parentIndex: 0, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e3'); +}); + +test('handleFindCommands click prefers semantic controls over matching containers', async () => { + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Later', 'click'], + flags: { findFirst: true }, + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + hittable: true, + rect: { x: 0, y: 0, width: 440, height: 956 }, + }, + { + index: 1, + ref: 'e2', + type: 'Element(5)', + label: 'Dialog', + hittable: true, + rect: { x: 60, y: 356, width: 320, height: 272 }, + parentIndex: 0, + }, + { + index: 2, + ref: 'e3', + type: 'ScrollView', + label: 'Later', + hittable: false, + rect: { x: 60, y: 548, width: 320, height: 80 }, + parentIndex: 1, + }, + { + index: 3, + ref: 'e4', + type: 'Other', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 288, height: 48 }, + parentIndex: 2, + }, + { + index: 4, + ref: 'e5', + type: 'Button', + label: 'Later', + hittable: false, + rect: { x: 76, y: 564, width: 140, height: 48 }, + parentIndex: 3, + }, + ], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0].positionals?.[0]).toBe('@e5'); +}); + test('handleFindCommands wait bypasses snapshot cache while Android freshness recovery is active', async () => { const sessionName = 'android-find-wait'; const session: SessionState = { diff --git a/src/daemon/handlers/__tests__/interaction-flags.test.ts b/src/daemon/handlers/__tests__/interaction-flags.test.ts index 5644dc84b..c88395aa7 100644 --- a/src/daemon/handlers/__tests__/interaction-flags.test.ts +++ b/src/daemon/handlers/__tests__/interaction-flags.test.ts @@ -9,4 +9,3 @@ test('unsupportedRefSnapshotFlags returns unsupported snapshot flags for @ref fl }); expect(unsupported).toEqual(['--depth', '--scope', '--raw']); }); - diff --git a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts index eed6d7af5..fc29098f4 100644 --- a/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts +++ b/src/daemon/handlers/__tests__/interaction-touch-targets.test.ts @@ -52,6 +52,19 @@ test('parseFillTarget reads selector text through shared fill codec', () => { }); }); +test('parseFillTarget preserves selector text whitespace', () => { + const parsed = parseFillTarget(['label="Command"', 'submit\n']); + + expect(parsed).toEqual({ + ok: true, + target: { + kind: 'selector', + selector: 'label="Command"', + }, + text: 'submit\n', + }); +}); + test('parseFillTarget rejects invalid coordinates instead of treating them as a point', () => { const parsed = parseFillTarget(['10', 'not-y', 'text']); diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 8d37ad9ad..a4e594141 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -417,6 +417,91 @@ test('click simple iOS id selector uses direct runner selector tap without snaps } }); +test('fill simple iOS id selector uses direct runner selector fill without snapshot coordinates', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-direct-selector-fill'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'filled', + x: 439.5, + y: 100.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['id="email"', 'ada@example.com'], + flags: { delayMs: 25 }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + expect(mockDispatch).toHaveBeenCalledTimes(1); + expect(mockDispatch.mock.calls[0]?.[1]).toBe('fill'); + expect(mockDispatch.mock.calls[0]?.[2]).toEqual(['ada@example.com']); + const context = mockDispatch.mock.calls[0]?.[4] as Record; + expect(context.directElementSelector).toEqual({ + key: 'id', + value: 'email', + raw: 'id="email"', + }); + expect(context.delayMs).toBe(25); + if (response?.ok) { + expect(response.data?.selector).toBe('id="email"'); + expect(response.data?.text).toBe('ada@example.com'); + } +}); + +test('click simple iOS selector forwards Maestro non-hittable coordinate fallback', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'ios-maestro-selector-fallback'; + sessionStore.set(sessionName, makeIosSession(sessionName, { appBundleId: 'com.example.app' })); + + mockDispatch.mockResolvedValue({ + message: 'tapped via non-hittable coordinate fallback', + x: 439.5, + y: 101.5, + referenceWidth: 440, + referenceHeight: 956, + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'click', + positionals: ['id="hiddenTestLogin"'], + flags: { maestro: { allowNonHittableCoordinateFallback: true } }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + const pressCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'press'); + expect(pressCalls.length).toBe(1); + expect((pressCalls[0]?.[4] as Record)?.directElementSelector).toEqual({ + key: 'id', + value: 'hiddenTestLogin', + raw: 'id="hiddenTestLogin"', + allowNonHittableCoordinateFallback: true, + }); + if (response?.ok) { + expect(response.data?.maestroNonHittableCoordinateFallbackAllowed).toBe(true); + expect(response.data?.maestroNonHittableCoordinateFallbackUsed).toBe(true); + expect(response.data?.maestroFallbackReason).toBe('non-hittable-coordinate'); + } +}); + test('click simple iOS id selector falls back to snapshot coordinates when direct tap fails', async () => { const sessionStore = makeSessionStore(); const sessionName = 'ios-direct-selector-fallback'; diff --git a/src/daemon/handlers/__tests__/session-open-target.test.ts b/src/daemon/handlers/__tests__/session-open-target.test.ts index d82605ad3..b07c6185e 100644 --- a/src/daemon/handlers/__tests__/session-open-target.test.ts +++ b/src/daemon/handlers/__tests__/session-open-target.test.ts @@ -31,4 +31,3 @@ test('inferAndroidPackageAfterOpen reads foreground package for Android URL open inferAndroidPackageAfterOpen(androidDevice, 'exp://127.0.0.1:8082', undefined), ).resolves.toBe('host.exp.exponent'); }); - diff --git a/src/daemon/handlers/__tests__/session-replay-vars.test.ts b/src/daemon/handlers/__tests__/session-replay-vars.test.ts index b50584763..d5fbc0d69 100644 --- a/src/daemon/handlers/__tests__/session-replay-vars.test.ts +++ b/src/daemon/handlers/__tests__/session-replay-vars.test.ts @@ -4,6 +4,7 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { AppError } from '../../../utils/errors.ts'; +import { runCmdBackground, type ExecBackgroundResult } from '../../../utils/exec.ts'; import type { DaemonRequest, DaemonResponse, SessionAction } from '../../types.ts'; import type { CommandFlags } from '../../../core/dispatch.ts'; import { SessionStore } from '../../session-store.ts'; @@ -32,6 +33,7 @@ type CapturedInvocation = { async function runReplayFixture(params: { label: string; script: string; + files?: Record; flags?: CommandFlags; invoke?: (req: DaemonRequest) => Promise; }): Promise<{ @@ -41,11 +43,15 @@ async function runReplayFixture(params: { scriptPath: string; }> { const root = fs.mkdtempSync(path.join(os.tmpdir(), `agent-device-replay-${params.label}-`)); + for (const [name, contents] of Object.entries(params.files ?? {})) { + fs.writeFileSync(path.join(root, name), contents); + } const scriptPath = path.join(root, 'flow.ad'); fs.writeFileSync(scriptPath, params.script); const calls: CapturedInvocation[] = []; - const defaultInvoke = async (req: DaemonRequest): Promise => { + const invoke = async (req: DaemonRequest): Promise => { calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (params.invoke) return await params.invoke(req); return { ok: true, data: {} }; }; const response = await runReplayScriptFile({ @@ -60,11 +66,39 @@ async function runReplayFixture(params: { sessionName: 's', logPath: path.join(root, 'log'), sessionStore: new SessionStore(path.join(root, 'state')), - invoke: params.invoke ?? defaultInvoke, + invoke, }); return { response, calls, root, scriptPath }; } +async function readFirstStdoutLine(process: ExecBackgroundResult): Promise { + return await new Promise((resolve, reject) => { + let stdout = ''; + const cleanup = (): void => { + clearTimeout(timer); + process.child.stdout?.off('data', onData); + process.child.off('exit', onExit); + }; + const timer = setTimeout(() => { + cleanup(); + reject(new Error('Timed out waiting for child process stdout.')); + }, 5000); + const onData = (chunk: Buffer | string): void => { + stdout += String(chunk); + const lineEnd = stdout.indexOf('\n'); + if (lineEnd === -1) return; + cleanup(); + resolve(stdout.slice(0, lineEnd)); + }; + const onExit = (): void => { + cleanup(); + reject(new Error('Child process exited before writing stdout.')); + }; + process.child.stdout?.on('data', onData); + process.child.on('exit', onExit); + }); +} + test('resolveReplayString substitutes variables', () => { const scope = buildReplayVarScope({ fileEnv: { APP: 'settings' } }); assert.equal(resolveReplayString('open ${APP}', scope, LOC), 'open settings'); @@ -401,11 +435,1189 @@ test('runReplayScriptFile applies CLI env overrides before Maestro compat mappin replayShellEnv: { AD_VAR_BUTTON_ID: 'shell-button' }, replayEnv: ['APP_ID=cli-app'], }, + invoke: async (req) => { + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'shell-button', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, }); assert.equal(response.ok, true); assert.deepEqual(calls[0]?.positionals, ['cli-app']); - assert.deepEqual(calls[1]?.positionals, ['id="shell-button"']); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['open', ['cli-app']], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile runs Maestro runScript in replay order and exposes output variables', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-runtime', + files: { + 'setup.js': ` +var res = {body: '{"appviewDid":"did:plc:test"}'} +output.result = SERVER_PATH + ':' + json(res.body).appviewDid +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript:', + ' file: ./setup.js', + ' env:', + ' SERVER_PATH: local', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['local:did:plc:test']]], + ); +}); + +test('runReplayScriptFile supports successful Maestro runScript http.post calls', async () => { + const serverScript = path.join( + fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-runscript-http-')), + 'server.cjs', + ); + fs.writeFileSync( + serverScript, + ` +const http = require('node:http'); +const server = http.createServer((req, res) => { + let body = ''; + req.setEncoding('utf8'); + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify({method: req.method, body})); + }); +}); +server.listen(0, '127.0.0.1', () => { + process.stdout.write(String(server.address().port) + '\\n'); +}); +`, + ); + const server = runCmdBackground(process.execPath, [serverScript], { allowFailure: true }); + const port = await readFirstStdoutLine(server); + + try { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-http-post', + files: { + 'setup.js': ` +var res = http.post('http://127.0.0.1:${port}/setup', {body: '{"ok":true}'}) +var parsed = json(res.body) +output.result = parsed.method + ':' + json(parsed.body).ok +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript: ./setup.js', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['POST:true']]], + ); + } finally { + server.child.kill(); + await server.wait.catch(() => undefined); + } +}); + +test('runReplayScriptFile strips prototype pollution keys from runScript json()', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-json-prototype-keys', + files: { + 'setup.js': ` +var parsed = json('{"safe":1,"__proto__":{"polluted":true},"constructor":{"polluted":true},"nested":{"prototype":{"polluted":true},"ok":2}}') +output.result = [ + Object.prototype.hasOwnProperty.call(parsed, '__proto__'), + Object.prototype.hasOwnProperty.call(parsed, 'constructor'), + Object.prototype.hasOwnProperty.call(parsed.nested, 'prototype'), + parsed.nested.ok +].join(':') +`, + }, + script: [ + 'appId: demo.app', + '---', + '- runScript: ./setup.js', + '- inputText: ${output.result}', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['type', ['false:false:false:2']]], + ); +}); + +test('runReplayScriptFile reports Maestro runScript failures at the runScript step', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-fail', + files: { + 'setup.js': `output.result = http.post('http://127.0.0.1:1').body`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /runScript failed/); + assert.match(response.error.message, /http\.post failed/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile explains empty Maestro runScript JSON bodies', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-empty-json', + files: { + 'setup.js': `output.result = json('').value`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /json\(\) received an empty body/); + assert.match(response.error.message, /setup server output/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile rejects Maestro runScript output keys containing dots', async () => { + const { response, calls } = await runReplayFixture({ + label: 'maestro-runscript-dotted-output', + files: { + 'setup.js': `output['nested.value'] = 'ambiguous'`, + }, + script: ['appId: demo.app', '---', '- runScript: ./setup.js', '- inputText: never', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro' }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /output key cannot contain/); + } + assert.equal(calls.length, 0); +}); + +test('runReplayScriptFile retries Maestro scrollUntilVisible with scroll probes', async () => { + const calls: CapturedInvocation[] = []; + let waitAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-scroll-until-visible', + script: [ + 'appId: demo.app', + '---', + '- scrollUntilVisible:', + ' element: Discover', + ' direction: UP', + ' timeout: 1200', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'scroll') return { ok: true, data: {} }; + if (req.command === 'find') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'find wait timed out' }, + }; + } + waitAttempts += 1; + if (waitAttempts === 3) return { ok: true, data: { waitedMs: 1100 } }; + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'wait timed out' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['up']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '500']], + ['find', ['Discover', 'wait', '500']], + ['scroll', ['up']], + ['wait', ['label="Discover" || text="Discover" || id="Discover"', '200']], + ], + ); +}); + +test('runReplayScriptFile lets Maestro tapOn use fuzzy visible text matching', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Discover people', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['130', '622']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile reuses successful Maestro visibility snapshot for following tapOn', async () => { + let snapshots = 0; + const { response, calls } = await runReplayFixture({ + label: 'maestro-assert-visible-tap-cache', + script: ['appId: demo.app', '---', '- assertVisible: Open feed', '- tapOn: Open feed', ''].join( + '\n', + ), + flags: { replayBackend: 'maestro', platform: 'android' }, + invoke: async (req) => { + if (req.command === 'snapshot') { + snapshots += 1; + return { + ok: true, + data: { + nodes: + snapshots === 1 + ? [ + { + index: 1, + label: 'Article', + rect: { x: 10, y: 100, width: 160, height: 44 }, + }, + { + index: 2, + label: 'Open feed', + rect: { x: 20, y: 180, width: 180, height: 48 }, + }, + ] + : [ + { + index: 1, + label: 'AppStack.tsx (42:7)', + rect: { x: 28, y: 1304, width: 1025, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['110', '204']], + ], + ); +}); + +test('runReplayScriptFile treats absent Maestro assertNotVisible targets as passing', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-absent', + script: ['appId: demo.app', '---', '- assertNotVisible: Archived banner', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'Selector did not match', + details: { command: 'is', reason: 'selector_not_found' }, + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + [ + 'is', + ['visible', 'label="Archived banner" || text="Archived banner" || id="Archived banner"'], + ], + [ + 'is', + ['visible', 'label="Archived banner" || text="Archived banner" || id="Archived banner"'], + ], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile propagates Maestro assertNotVisible infrastructure failures', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-infra-fail', + script: ['appId: demo.app', '---', '- assertNotVisible: Archived banner', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Snapshot capture failed' }, + }; + }, + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.match(response.error.message, /Replay failed at step 1/); + assert.match(response.error.message, /Snapshot capture failed/); + } + assert.equal(calls.length, 1); +}); + +test('runReplayScriptFile waits briefly for Maestro assertNotVisible to stabilize', async () => { + const calls: CapturedInvocation[] = []; + let visibleChecks = 0; + const { response } = await runReplayFixture({ + label: 'maestro-assert-not-visible-stable', + script: ['appId: demo.app', '---', '- assertNotVisible: Archived banner', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + visibleChecks += 1; + if (visibleChecks === 1) return { ok: true, data: { pass: true } }; + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'is visible failed', + details: { command: 'is', reason: 'predicate_failed' }, + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.equal(calls.length, 3); +}); + +test('runReplayScriptFile retries Maestro fuzzy tapOn without raw selector fallback', async () => { + const calls: CapturedInvocation[] = []; + let snapshotAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-fuzzy-retry', + script: ['appId: demo.app', '---', '- tapOn: Discover', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + snapshotAttempts += 1; + return { + ok: true, + data: { + nodes: + snapshotAttempts === 1 + ? [] + : [ + { + index: 1, + label: 'Discover people', + rect: { x: 10, y: 600, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['snapshot', []], + ['click', ['130', '622']], + ], + ); +}); + +test('runReplayScriptFile lets optional Maestro fuzzy tapOn click first visible match', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-visible-text-optional-first-match', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' text: Later', + ' optional: true', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Maybe Later', + rect: { x: 100, y: 700, width: 240, height: 44 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['220', '722']], + ], + ); +}); + +test('runReplayScriptFile resolves Maestro percentage point taps from snapshot size', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-point-percent', + script: ['appId: demo.app', '---', '- tapOn:', ' point: 20%,20%', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 1000, height: 2000 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['200', '400']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile retries Maestro id tapOn through snapshot coordinates', async () => { + const calls: CapturedInvocation[] = []; + let snapshotAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-tap-on-retry', + script: ['appId: demo.app', '---', '- tapOn:', ' id: delayedButton', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + snapshotAttempts += 1; + if (snapshotAttempts === 3) { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'delayedButton', + rect: { x: 20, y: 40, width: 120, height: 44 }, + }, + ], + }, + }; + } + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + } + if (req.command === 'click') return { ok: true, data: {} }; + return { + ok: false, + error: { code: 'ELEMENT_NOT_FOUND', message: 'element not found' }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['snapshot', []], + ['snapshot', []], + ['click', ['80', '62']], + ], + ); +}); + +test('runReplayScriptFile resolves Maestro tapOn index and childOf from snapshots', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-index-childof', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: childActionButton', + ' childOf:', + ' id: parent-row-secondary', + '- tapOn:', + ' id: overflowButton', + ' index: 1', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { index: 1, identifier: 'parent-row-primary' }, + { + index: 2, + parentIndex: 1, + identifier: 'childActionButton', + rect: { x: 10, y: 10, width: 40, height: 20 }, + }, + { index: 10, identifier: 'parent-row-secondary' }, + { + index: 11, + parentIndex: 10, + identifier: 'childActionButton', + rect: { x: 20, y: 120, width: 40, height: 20 }, + }, + { + index: 20, + identifier: 'overflowButton', + rect: { x: 100, y: 200, width: 40, height: 20 }, + }, + { + index: 21, + identifier: 'overflowButton', + rect: { x: 200, y: 300, width: 40, height: 20 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['40', '130']], + ['snapshot', []], + ['click', ['220', '310']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile lets snapshot id tap handle Maestro one-point edge controls', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-edge-rect', + script: ['appId: demo.app', '---', '- tapOn:', ' id: hiddenTestLogin', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'hiddenTestLogin', + rect: { x: 0, y: 0, width: 1, height: 1 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['0', '0']], + ], + ); +}); + +test('runReplayScriptFile keeps Maestro text-entry tapOn on the snapshot path', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-tap-input-text-snapshot', + script: [ + 'appId: demo.app', + '---', + '- tapOn:', + ' id: editableNameInput', + '- inputText: Saved list', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + identifier: 'editableNameInput', + rect: { x: 20, y: 100, width: 200, height: 40 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['click', ['120', '120']], + ['type', ['Saved list']], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile resolves Maestro swipe.label from a labeled element rect', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-swipe-label', + script: [ + 'appId: demo.app', + '---', + '- swipe:', + ' label: Thread body', + ' direction: UP', + ' duration: 400', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Thread body', + rect: { x: 10, y: 100, width: 200, height: 300 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['swipe', ['110', '250', '110', '8', '400']], + ], + ); +}); + +test('runReplayScriptFile resolves Maestro screen swipes from the snapshot frame', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-screen-swipe', + script: [ + 'appId: demo.app', + '---', + '- swipe:', + ' direction: LEFT', + ' duration: 300', + '- swipe:', + ' start: 90%,50%', + ' end: 10%,50%', + ' duration: 300', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['swipe', ['320', '400', '80', '400', '300']], + ['swipe', ['360', '400', '40', '400', '300']], + ], + ); +}); + +test('runReplayScriptFile maps Maestro enter to keyboard enter', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['keyboard', ['enter']]], + ); +}); + +test('runReplayScriptFile waits for Maestro animation snapshots to stabilize', async () => { + const calls: CapturedInvocation[] = []; + let snapshots = 0; + const { response } = await runReplayFixture({ + label: 'maestro-wait-animation-stable', + script: ['appId: demo.app', '---', '- waitForAnimationToEnd', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + snapshots += 1; + const y = snapshots === 1 ? 100 : 120; + return { + ok: true, + data: { + nodes: [ + { + index: 1, + label: 'Animating', + rect: { x: 10, y, width: 100, height: 40 }, + }, + ], + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['snapshot', []], + ['snapshot', []], + ], + ); + assert.equal(calls[0]?.flags?.noRecord, true); +}); + +test('runReplayScriptFile falls back to newline type when keyboard enter is unsupported', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-press-enter-fallback', + script: ['appId: demo.app', '---', '- pressKey: Enter', ''].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'keyboard') { + return { ok: false, error: { code: 'UNSUPPORTED_OPERATION', message: 'unsupported' } }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['keyboard', ['enter']], + ['type', ['\n']], + ], + ); +}); + +test('runReplayScriptFile skips Maestro runFlow.when.visible commands when absent', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-skip', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ], + }, + }; + } + return { + ok: false, + error: { + code: 'COMMAND_FAILED', + message: 'not visible', + details: { command: 'is', reason: 'selector_not_found' }, + }, + }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [['snapshot', []]], + ); +}); + +test('runReplayScriptFile retries Maestro retry commands until they pass', async () => { + const calls: CapturedInvocation[] = []; + let openAttempts = 0; + const { response } = await runReplayFixture({ + label: 'maestro-retry', + script: [ + 'appId: demo.app', + '---', + '- retry:', + ' maxRetries: 2', + ' commands:', + ' - openLink:', + ' link: demo://details', + ' - extendedWaitUntil:', + ' visible: Article', + ' timeout: 1', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'open') openAttempts += 1; + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + ...(openAttempts > 1 + ? [ + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'statictext', + label: 'Article', + rect: { x: 16, y: 100, width: 120, height: 24 }, + }, + ] + : []), + ], + }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.filter((call) => call.command === 'open').map((call) => [call.command, call.positionals]), + [ + ['open', ['demo://details']], + ['open', ['demo://details']], + ], + ); + assert.equal(calls.filter((call) => call.command === 'snapshot').length > 1, true); +}); + +test('runReplayScriptFile propagates Maestro runFlow.when runtime errors', async () => { + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-runtime-error', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async () => ({ + ok: false, + error: { code: 'UNKNOWN', message: 'fetch failed' }, + }), + }); + + assert.equal(response.ok, false); + if (!response.ok) { + assert.equal(response.error.code, 'UNKNOWN'); + assert.match(response.error.message, /fetch failed/); + } +}); + +test('runReplayScriptFile runs Maestro runFlow.when.visible commands when present', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-visible-run', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Continue', + ' commands:', + ' - tapOn: Continue', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'button', + label: 'Continue', + rect: { x: 16, y: 100, width: 120, height: 44 }, + }, + ], + }, + }; + } + if (req.command === 'click') { + return { + ok: false, + error: { code: 'COMMAND_FAILED', message: 'Selector did not match' }, + }; + } + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['snapshot', []], + ['click', ['76', '122']], + ['find', ['Continue', 'click']], + ], + ); +}); + +test('runReplayScriptFile runs nested Maestro runtime commands inside runFlow.when', async () => { + const calls: CapturedInvocation[] = []; + const { response } = await runReplayFixture({ + label: 'maestro-run-flow-when-nested-runtime', + script: [ + 'appId: demo.app', + '---', + '- runFlow:', + ' when:', + ' visible: Feed', + ' commands:', + ' - scrollUntilVisible:', + ' element: Done', + ' direction: DOWN', + ' timeout: 500', + '', + ].join('\n'), + flags: { replayBackend: 'maestro' }, + invoke: async (req) => { + calls.push({ command: req.command, positionals: req.positionals, flags: req.flags }); + if (req.command === 'snapshot') { + return { + ok: true, + data: { + nodes: [ + { + index: 0, + type: 'application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'statictext', + label: 'Feed', + rect: { x: 16, y: 100, width: 120, height: 24 }, + }, + ], + }, + }; + } + if (req.command === 'wait') return { ok: true, data: { found: true } }; + return { ok: true, data: {} }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual( + calls.map((call) => [call.command, call.positionals]), + [ + ['snapshot', []], + ['wait', ['label="Done" || text="Done" || id="Done"', '500']], + ], + ); }); test('runReplayScriptFile reads shell env from request (client-collected), not daemon process.env', async () => { diff --git a/src/daemon/handlers/__tests__/session-test-discovery.test.ts b/src/daemon/handlers/__tests__/session-test-discovery.test.ts index dfc4ffa6a..f0c572537 100644 --- a/src/daemon/handlers/__tests__/session-test-discovery.test.ts +++ b/src/daemon/handlers/__tests__/session-test-discovery.test.ts @@ -51,7 +51,29 @@ test('discoverReplayTestEntries rejects empty post-filter suites', () => { (error: unknown) => error instanceof AppError && error.code === 'INVALID_ARGS' && - error.message === 'No .ad tests matched for --platform android.', + error.message === 'No replay tests matched for --platform android.', ); }); +test('discoverReplayTestEntries includes Maestro yaml flows for Maestro test suites', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-device-test-discovery-maestro-')); + fs.writeFileSync(path.join(root, '01-flow.yaml'), 'appId: demo\n---\n- launchApp\n'); + fs.writeFileSync(path.join(root, '02-flow.yml'), 'appId: demo\n---\n- launchApp\n'); + fs.writeFileSync(path.join(root, '03-flow.ad'), 'open "Demo"\n'); + + const entries = discoverReplayTestEntries({ + inputs: [root], + cwd: root, + platformFilter: 'android', + replayBackend: 'maestro', + }); + + assert.deepEqual( + entries.map((entry) => path.basename(entry.path)), + ['01-flow.yaml', '02-flow.yml', '03-flow.ad'], + ); + assert.deepEqual( + entries.map((entry) => entry.kind), + ['run', 'run', 'run'], + ); +}); diff --git a/src/daemon/handlers/__tests__/session-test-runtime.test.ts b/src/daemon/handlers/__tests__/session-test-runtime.test.ts index a9bbd969d..e9e72ea76 100644 --- a/src/daemon/handlers/__tests__/session-test-runtime.test.ts +++ b/src/daemon/handlers/__tests__/session-test-runtime.test.ts @@ -47,4 +47,3 @@ test('runReplayTestAttempt keeps cancellation active until a timed-out replay se expect(isRequestCanceled('req-timeout-open')).toBe(false); }); - diff --git a/src/daemon/handlers/__tests__/session.test.ts b/src/daemon/handlers/__tests__/session.test.ts index 9773e19ed..61d5470ac 100644 --- a/src/daemon/handlers/__tests__/session.test.ts +++ b/src/daemon/handlers/__tests__/session.test.ts @@ -4322,7 +4322,7 @@ test('test returns invalid args when no replay scripts match the platform filter invoke: noopInvoke, }); - assertInvalidArgsMessage(response, 'No .ad tests matched for --platform android.'); + assertInvalidArgsMessage(response, 'No replay tests matched for --platform android.'); }); test('test rejects duplicate replay test metadata in the context header', async () => { diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 42f2c9104..7efc4988d 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -6,7 +6,11 @@ import type { DaemonRequest, DaemonResponse } from '../types.ts'; import { SessionStore } from '../session-store.ts'; import { contextFromFlags } from '../context.ts'; import { ensureDeviceReady } from '../device-ready.ts'; -import { extractNodeText, findNearestHittableAncestor } from '../snapshot-processing.ts'; +import { extractNodeText } from '../snapshot-processing.ts'; +import { + resolveActionableTouchNode, + resolveActionableTouchResolution, +} from '../../commands/interaction-targeting.ts'; import { readTextForNode } from './interaction-read.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; @@ -37,6 +41,10 @@ type ResolvedMatch = { actionFlags: Record; }; +type FindMatchResult = + | { ok: true; node: SnapshotState['nodes'][number] } + | { ok: false; response: DaemonResponse }; + export async function handleFindCommands(params: { req: DaemonRequest; sessionName: string; @@ -67,8 +75,7 @@ export async function handleFindCommands(params: { }); if (runtimeResponse) return runtimeResponse; const session = sessionStore.get(sessionName); - const isReadOnly = - action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs'; + const isReadOnly = isReadOnlyFindAction(action); if (!session && !isReadOnly) { return errorResponse('SESSION_NOT_FOUND', 'No active session. Run open first.'); } @@ -76,9 +83,10 @@ export async function handleFindCommands(params: { if (!session) { await ensureDeviceReady(device); } - const scope = shouldScopeFind(locator) ? query : undefined; - const requiresRect = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; + const requiresRect = findActionRequiresRect(action); + // Interaction targets need the full compact tree so duplicate labels can be + // resolved against viewport visibility before an off-screen subtree wins. + const scope = shouldScopeFind(locator) && !requiresRect ? query : undefined; const interactiveOnly = requiresRect; let lastSnapshotAt = 0; let lastNodes: SnapshotState['nodes'] | null = null; @@ -134,29 +142,16 @@ export async function handleFindCommands(params: { } const { nodes } = await fetchNodes(); - const bestMatches = findBestMatchesByLocator(nodes, locator, query, { - requireRect: requiresRect, + const matchResult = resolveFindMatch({ + nodes, + locator, + query, + requiresRect, + flags: req.flags, }); - - if (requiresRect && bestMatches.matches.length > 1) { - if (req.flags?.findFirst) { - bestMatches.matches = [bestMatches.matches[0]]; - } else if (req.flags?.findLast) { - bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; - } else { - return buildAmbiguousMatchError(bestMatches.matches, locator, query); - } - } - - const node = bestMatches.matches[0] ?? null; - if (!node) { - return errorResponse('COMMAND_FAILED', 'find did not match any element'); - } - - const resolvedNode = - action === 'click' || action === 'focus' || action === 'fill' || action === 'type' - ? (findNearestHittableAncestor(nodes, node) ?? node) - : node; + if (!matchResult.ok) return matchResult.response; + const node = matchResult.node; + const resolvedNode = requiresRect ? resolveInteractiveMatchNode(nodes, node) : node; const ref = `@${resolvedNode.ref}`; const actionFlags = { ...(req.flags ?? {}), noRecord: true }; const match: ResolvedMatch = { node, resolvedNode, ref, nodes, actionFlags }; @@ -177,6 +172,128 @@ export async function handleFindCommands(params: { // --- Per-action handlers --- +function isReadOnlyFindAction(action: string): boolean { + return ( + action === 'exists' || action === 'wait' || action === 'get_text' || action === 'get_attrs' + ); +} + +function findActionRequiresRect(action: string): boolean { + return action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; +} + +function resolveFindMatch(params: { + nodes: SnapshotState['nodes']; + locator: FindLocator; + query: string; + requiresRect: boolean; + flags: DaemonRequest['flags']; +}): FindMatchResult { + const { nodes, locator, query, requiresRect, flags } = params; + const bestMatches = findBestMatchesByLocator(nodes, locator, query, { + requireRect: requiresRect, + }); + if (requiresRect) { + bestMatches.matches = preferOnscreenMatches(bestMatches.matches, nodes); + } + + if (requiresRect && bestMatches.matches.length > 1) { + if (flags?.findFirst) { + bestMatches.matches = [bestMatches.matches[0]]; + } else if (flags?.findLast) { + bestMatches.matches = [bestMatches.matches[bestMatches.matches.length - 1]]; + } else { + return { ok: false, response: buildAmbiguousMatchError(bestMatches.matches, locator, query) }; + } + } + + const node = bestMatches.matches[0] ?? null; + if (!node) { + return { + ok: false, + response: errorResponse('COMMAND_FAILED', 'find did not match any element'), + }; + } + return { ok: true, node }; +} + +function preferOnscreenMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + const viewport = nodes[0]?.rect; + if (!viewport) return matches; + const onscreen = matches.filter((node) => { + if (!node.rect) return false; + const center = centerOfRect(node.rect); + return ( + center.x >= viewport.x && + center.x <= viewport.x + viewport.width && + center.y >= viewport.y && + center.y <= viewport.y + viewport.height + ); + }); + return rankInteractiveMatches(onscreen.length > 0 ? onscreen : matches, nodes); +} + +function rankInteractiveMatches( + matches: SnapshotState['nodes'], + nodes: SnapshotState['nodes'], +): SnapshotState['nodes'] { + if (matches.length < 2) return matches; + return matches + .map((node, index) => ({ node, index, score: interactiveMatchScore(node, nodes) })) + .sort((left, right) => { + if (right.score !== left.score) return right.score - left.score; + return rectArea(left.node) - rectArea(right.node) || left.index - right.index; + }) + .map((entry) => entry.node); +} + +function interactiveMatchScore( + node: SnapshotState['nodes'][number], + nodes: SnapshotState['nodes'], +): number { + const resolution = resolveActionableTouchResolution(nodes, node); + if (resolution.reason === 'semantic-target' && resolution.node.rect) return 4; + if (resolution.reason === 'same-rect-descendant' && resolution.node.rect) return 4; + if ( + resolution.reason === 'hittable-ancestor' && + resolution.node.rect && + !isRootInteractionContainer(resolution.node, nodes[0]) + ) { + return 2; + } + if (node.hittable && node.rect && !isRootInteractionContainer(node, nodes[0])) return 3; + return node.rect ? 1 : 0; +} + +function rectArea(node: SnapshotState['nodes'][number]): number { + return node.rect ? node.rect.width * node.rect.height : Number.POSITIVE_INFINITY; +} + +function resolveInteractiveMatchNode( + nodes: SnapshotState['nodes'], + node: SnapshotState['nodes'][number], +): SnapshotState['nodes'][number] { + return resolveActionableTouchNode(nodes, node); +} + +function isRootInteractionContainer( + node: SnapshotState['nodes'][number], + root: SnapshotState['nodes'][number] | undefined, +): boolean { + if (!root?.rect || !node.rect) return false; + const type = node.type?.toLowerCase() ?? ''; + if (!type.includes('application') && !type.includes('window')) return false; + return ( + node.rect.x === root.rect.x && + node.rect.y === root.rect.y && + node.rect.width === root.rect.width && + node.rect.height === root.rect.height + ); +} + async function handleFindWait( ctx: FindContext, fetchNodes: () => Promise<{ nodes: SnapshotState['nodes'] }>, @@ -266,7 +383,11 @@ async function handleFindClick(ctx: FindContext, match: ResolvedMatch): Promise< flags: match.actionFlags, }); if (!response.ok) return response; - const matchCoords = match.resolvedNode.rect ? centerOfRect(match.resolvedNode.rect) : null; + const matchCoords = match.resolvedNode.rect + ? centerOfRect(match.resolvedNode.rect) + : match.node.rect + ? centerOfRect(match.node.rect) + : null; const matchData: Record = { ref: match.ref, locator, query }; if (matchCoords) { matchData.x = matchCoords.x; @@ -312,7 +433,35 @@ async function handleFindFill( } async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; + const response = await dispatchFocusForFindMatch(ctx, match); + if (!response.ok) return response; + recordFindAction(ctx, match, 'focus'); + return response; +} + +async function handleFindType( + ctx: FindContext, + match: ResolvedMatch, + value: string | undefined, +): Promise { + const { req, device, logPath, session } = ctx; + if (!value) { + return errorResponse('INVALID_ARGS', 'find type requires text'); + } + const focusResponse = await dispatchFocusForFindMatch(ctx, match); + if (!focusResponse.ok) return focusResponse; + const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { + ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), + }); + recordFindAction(ctx, match, 'type'); + return { ok: true, data: response ?? { ref: match.ref } }; +} + +async function dispatchFocusForFindMatch( + ctx: FindContext, + match: ResolvedMatch, +): Promise { + const { req, device, logPath, session } = ctx; const coords = match.node.rect ? centerOfRect(match.node.rect) : null; if (!coords) { return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); @@ -326,45 +475,19 @@ async function handleFindFocus(ctx: FindContext, match: ResolvedMatch): Promise< ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), }, ); - if (session) { - sessionStore.recordAction(session, { - command, - positionals: req.positionals ?? [], - flags: req.flags ?? {}, - result: { ref: match.ref, action: 'focus' }, - }); - } return { ok: true, data: response ?? { ref: match.ref } }; } -async function handleFindType( - ctx: FindContext, - match: ResolvedMatch, - value: string | undefined, -): Promise { - const { req, sessionStore, session, device, command, logPath } = ctx; - if (!value) { - return errorResponse('INVALID_ARGS', 'find type requires text'); - } - const coords = match.node.rect ? centerOfRect(match.node.rect) : null; - if (!coords) { - return errorResponse('COMMAND_FAILED', 'matched element has no bounds'); - } - await dispatchCommand(device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); - const response = await dispatchCommand(device, 'type', [value], req.flags?.out, { - ...contextFromFlags(logPath, req.flags, session?.appBundleId, session?.trace?.outPath), - }); +function recordFindAction(ctx: FindContext, match: ResolvedMatch, action: string): void { + const { req, sessionStore, session, command } = ctx; if (session) { sessionStore.recordAction(session, { command, positionals: req.positionals ?? [], flags: req.flags ?? {}, - result: { ref: match.ref, action: 'type' }, + result: { ref: match.ref, action }, }); } - return { ok: true, data: response ?? { ref: match.ref } }; } // --- Helpers --- diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index 264302ef4..b22beb409 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -110,8 +110,9 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { ), }; } - const text = parsed.text.trim(); - if (!text) { + // Preserve payload whitespace (for example Maestro/keyboard-enter newlines) + // while still rejecting selector fills that contain only whitespace. + if (!parsed.text.trim()) { return { ok: false, response: errorResponse('INVALID_ARGS', 'fill requires text after selector'), @@ -120,7 +121,7 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { return { ok: true, target: { kind: 'selector', selector: parsed.target.selector }, - text, + text: parsed.text, }; } diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 494722171..8aa6df4ef 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -258,7 +258,14 @@ function readDirectIosSelectorTapTarget(params: { if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; if (hasNonDefaultClickOptions(flags)) return null; - return readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), + }; } function hasNonDefaultClickOptions(flags: CommandFlags | undefined): boolean { @@ -277,11 +284,44 @@ async function dispatchDirectIosSelectorTap( session: SessionState, selector: DirectIosSelectorTarget, ): Promise { + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'press', + positionals: [], + extra: { selector: selector.raw }, + fallbackPhase: 'ios_direct_selector_tap_fallback', + }); +} + +async function dispatchDirectIosSelectorInteraction(params: { + params: InteractionHandlerParams; + session: SessionState; + selector: DirectIosSelectorTarget; + command: 'press' | 'fill'; + positionals: string[]; + extra: Record; + fallbackPhase: string; +}): Promise { + const { + params: handlerParams, + session, + selector, + command, + positionals, + extra, + fallbackPhase, + } = params; const actionStartedAt = Date.now(); try { const data = - (await dispatchCommand(session.device, 'press', [], params.req.flags?.out, { - ...params.contextFromFlags(params.req.flags, session.appBundleId, session.trace?.outPath), + (await dispatchCommand(session.device, command, positionals, handlerParams.req.flags?.out, { + ...handlerParams.contextFromFlags( + handlerParams.req.flags, + session.appBundleId, + session.trace?.outPath, + ), directElementSelector: selector, surface: session.surface, })) ?? {}; @@ -293,15 +333,16 @@ async function dispatchDirectIosSelectorTap( fallbackY: point.y, referenceFrame: readReferenceFrameFromDirectSelectorTapResult(data), extra: { - selector: selector.raw, + ...extra, + ...directIosSelectorFallbackDetails(selector, data), }, }); return finalizeTouchInteraction({ session, - sessionStore: params.sessionStore, - command: params.req.command, - positionals: params.req.positionals ?? [], - flags: params.req.flags, + sessionStore: handlerParams.sessionStore, + command: handlerParams.req.command, + positionals: handlerParams.req.positionals ?? [], + flags: handlerParams.req.flags, result: responseData, responseData, actionStartedAt, @@ -313,7 +354,7 @@ async function dispatchDirectIosSelectorTap( } emitDiagnostic({ level: 'debug', - phase: 'ios_direct_selector_tap_fallback', + phase: fallbackPhase, data: { selector: selector.raw, error: error instanceof Error ? error.message : String(error), @@ -323,6 +364,19 @@ async function dispatchDirectIosSelectorTap( } } +function directIosSelectorFallbackDetails( + selector: DirectIosSelectorTarget, + data: Record, +): Record { + if (!selector.allowNonHittableCoordinateFallback) return {}; + const used = data.message === 'tapped via non-hittable coordinate fallback'; + return { + maestroNonHittableCoordinateFallbackAllowed: true, + maestroNonHittableCoordinateFallbackUsed: used, + ...(used ? { maestroFallbackReason: 'non-hittable-coordinate' } : {}), + }; +} + function readPointFromDirectSelectorTapResult(data: Record): { x: number; y: number; @@ -367,6 +421,20 @@ async function dispatchFillViaRuntime( if (invalidRefFlagsResponse) return invalidRefFlagsResponse; await refreshAndroidRefSnapshotIfFreshnessActive(params, session); } + const directSelector = readDirectIosSelectorFillTarget({ + session, + target: parsedTarget.target, + flags: req.flags, + }); + if (directSelector) { + const directResponse = await dispatchDirectIosSelectorFill( + params, + session, + directSelector, + parsedTarget.text, + ); + if (directResponse) return directResponse; + } return await dispatchRuntimeInteraction(params, { run: async (runtime) => @@ -408,6 +476,40 @@ async function dispatchFillViaRuntime( }); } +function readDirectIosSelectorFillTarget(params: { + session: SessionState; + target: InteractionTarget; + flags: CommandFlags | undefined; +}): DirectIosSelectorTarget | null { + const { session, target, flags } = params; + if (target.kind !== 'selector') return null; + const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); + if (!selector) return null; + return { + ...selector, + ...(flags?.maestro?.allowNonHittableCoordinateFallback + ? { allowNonHittableCoordinateFallback: true } + : {}), + }; +} + +async function dispatchDirectIosSelectorFill( + params: InteractionHandlerParams, + session: SessionState, + selector: DirectIosSelectorTarget, + text: string, +): Promise { + return await dispatchDirectIosSelectorInteraction({ + params, + session, + selector, + command: 'fill', + positionals: [text], + extra: { selector: selector.raw, text }, + fallbackPhase: 'ios_direct_selector_fill_fallback', + }); +} + async function dispatchRuntimeInteraction< TResult extends PressCommandResult | FillCommandResult | LongPressCommandResult, >( diff --git a/src/daemon/handlers/session-close.ts b/src/daemon/handlers/session-close.ts index b03c6ee64..3f1c1037a 100644 --- a/src/daemon/handlers/session-close.ts +++ b/src/daemon/handlers/session-close.ts @@ -17,6 +17,7 @@ import { IOS_SIMULATOR_POST_CLOSE_SETTLE_MS, isAndroidEmulator, isIosSimulator, + resolveCommandDevice, settleIosSimulator, } from './session-device-utils.ts'; import { errorResponse } from './response.ts'; @@ -123,7 +124,7 @@ export async function handleCloseCommand(params: { const { req, sessionName, logPath, sessionStore } = params; const session = sessionStore.get(sessionName); if (!session) { - return errorResponse('SESSION_NOT_FOUND', 'No active session'); + return await closeWithoutSession(req, logPath); } if (session.appLog) { await stopAppLog(session.appLog); @@ -188,3 +189,25 @@ export async function handleCloseCommand(params: { } return { ok: true, data: { session: sessionName, ...successText(`Closed: ${sessionName}`) } }; } + +async function closeWithoutSession(req: DaemonRequest, logPath: string): Promise { + if (!req.positionals || req.positionals.length === 0) { + return errorResponse('SESSION_NOT_FOUND', 'No active session'); + } + const device = await resolveCommandDevice({ + session: undefined, + flags: req.flags, + ensureReady: true, + }); + await dispatchCommand(device, 'close', req.positionals, req.flags?.out, { + ...contextFromFlags(logPath, req.flags), + }); + await settleIosSimulator(device, IOS_SIMULATOR_POST_CLOSE_SETTLE_MS); + return { + ok: true, + data: { + app: req.positionals[0], + ...successText(`Closed: ${req.positionals[0]}`), + }, + }; +} diff --git a/src/daemon/handlers/session-replay-action-runtime.ts b/src/daemon/handlers/session-replay-action-runtime.ts new file mode 100644 index 000000000..3ab0868df --- /dev/null +++ b/src/daemon/handlers/session-replay-action-runtime.ts @@ -0,0 +1,148 @@ +import fs from 'node:fs'; +import type { CommandFlags } from '../../core/dispatch.ts'; +import { + mergeReplayVarScopeValues, + resolveReplayAction, + type ReplayVarScope, +} from '../../replay/vars.ts'; +import type { DaemonRequest, DaemonResponse, SessionAction } from '../types.ts'; +import { mergeParentFlags } from './handler-utils.ts'; +import { invokeMaestroRuntimeCommand } from '../../compat/maestro/runtime.ts'; + +type ReplayBaseRequest = Omit; + +type ReplayActionInvoker = (params: { + action: SessionAction; + line: number; + step: number; +}) => Promise; + +export async function invokeReplayAction(params: { + req: DaemonRequest; + sessionName: string; + action: SessionAction; + scope: ReplayVarScope; + filePath: string; + line: number; + step: number; + tracePath?: string; + invoke: (req: DaemonRequest) => Promise; +}): Promise { + const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; + const resolved = resolveReplayAction(action, scope, { file: filePath, line }); + const invokeNestedReplayAction: ReplayActionInvoker = (nested) => + invokeReplayAction({ + req, + sessionName, + action: nested.action, + scope, + filePath, + line: nested.line, + step: nested.step, + tracePath, + invoke, + }); + const startedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_start', + ts: new Date(startedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + positionals: resolved.positionals ?? [], + }); + + const response = await invokeResolvedReplayAction({ + req, + sessionName, + resolved, + scope, + line, + step, + invoke, + invokeReplayAction: invokeNestedReplayAction, + }); + + const finishedAt = Date.now(); + appendReplayTraceEvent(tracePath, { + type: 'replay_action_stop', + ts: new Date(finishedAt).toISOString(), + replayPath: filePath, + line, + step, + command: resolved.command, + ok: response.ok, + durationMs: finishedAt - startedAt, + errorCode: response.ok ? undefined : response.error.code, + }); + return response; +} + +async function invokeResolvedReplayAction(params: { + req: DaemonRequest; + sessionName: string; + resolved: SessionAction; + scope: ReplayVarScope; + line: number; + step: number; + invoke: (req: DaemonRequest) => Promise; + invokeReplayAction: ReplayActionInvoker; +}): Promise { + const { req, sessionName, resolved, scope, line, step, invoke, invokeReplayAction } = params; + const flags = buildReplayActionFlags(req.flags, resolved.flags); + const baseReq: ReplayBaseRequest = { + token: req.token, + session: sessionName, + flags, + runtime: resolved.runtime, + meta: req.meta, + }; + const response = + (await invokeMaestroRuntimeCommand({ + command: resolved.command, + baseReq, + positionals: resolved.positionals ?? [], + batchSteps: resolved.flags?.batchSteps, + scope, + line, + step, + invoke, + invokeReplayAction, + })) ?? + (await invoke({ + ...baseReq, + command: resolved.command, + positionals: resolved.positionals ?? [], + })); + if (response.ok) { + const outputEnv = readReplayOutputEnv(response.data); + if (outputEnv) mergeReplayVarScopeValues(scope, outputEnv); + } + return response; +} + +function readReplayOutputEnv(data: unknown): Record | null { + if (!data || typeof data !== 'object') return null; + const raw = (data as { outputEnv?: unknown }).outputEnv; + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null; + const entries = Object.entries(raw).filter( + (entry): entry is [string, string] => typeof entry[1] === 'string', + ); + return entries.length > 0 ? Object.fromEntries(entries) : null; +} + +function appendReplayTraceEvent( + tracePath: string | undefined, + event: Record, +): void { + if (!tracePath) return; + fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); +} + +function buildReplayActionFlags( + parentFlags: CommandFlags | undefined, + actionFlags: SessionAction['flags'] | undefined, +): CommandFlags { + return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); +} diff --git a/src/daemon/handlers/session-replay-runtime.ts b/src/daemon/handlers/session-replay-runtime.ts index 12ff2cee0..37c59e8e4 100644 --- a/src/daemon/handlers/session-replay-runtime.ts +++ b/src/daemon/handlers/session-replay-runtime.ts @@ -8,16 +8,14 @@ import { SessionStore } from '../session-store.ts'; import { type ReplayScriptMetadata, writeReplayScript } from '../../replay/script.ts'; import { healReplayAction } from './session-replay-heal.ts'; import { formatScriptActionSummary } from '../../replay/script-utils.ts'; -import { mergeParentFlags } from './handler-utils.ts'; import { errorResponse } from './response.ts'; +import { invokeReplayAction } from './session-replay-action-runtime.ts'; import { buildReplayVarScope, collectReplayShellEnv, parseReplayCliEnvEntries, readReplayCliEnvEntries, readReplayShellEnvSource, - resolveReplayAction, - type ReplayVarScope, } from '../../replay/vars.ts'; // fallow-ignore-next-line complexity @@ -157,61 +155,6 @@ export async function runReplayScriptFile(params: { } } -async function invokeReplayAction(params: { - req: DaemonRequest; - sessionName: string; - action: SessionAction; - scope: ReplayVarScope; - filePath: string; - line: number; - step: number; - tracePath?: string; - invoke: (req: DaemonRequest) => Promise; -}): Promise { - const { req, sessionName, action, scope, filePath, line, step, tracePath, invoke } = params; - const resolved = resolveReplayAction(action, scope, { file: filePath, line }); - const startedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_start', - ts: new Date(startedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - positionals: resolved.positionals ?? [], - }); - const response = await invoke({ - token: req.token, - session: sessionName, - command: resolved.command, - positionals: resolved.positionals ?? [], - flags: buildReplayActionFlags(req.flags, resolved.flags), - runtime: resolved.runtime, - meta: req.meta, - }); - const finishedAt = Date.now(); - appendReplayTraceEvent(tracePath, { - type: 'replay_action_stop', - ts: new Date(finishedAt).toISOString(), - replayPath: filePath, - line, - step, - command: resolved.command, - ok: response.ok, - durationMs: finishedAt - startedAt, - errorCode: response.ok ? undefined : response.error.code, - }); - return response; -} - -function appendReplayTraceEvent( - tracePath: string | undefined, - event: Record, -): void { - if (!tracePath) return; - fs.appendFileSync(tracePath, `${JSON.stringify(event)}\n`); -} - // fallow-ignore-next-line complexity function buildReplayBuiltinVars(params: { req: DaemonRequest; @@ -255,7 +198,7 @@ function buildReplayMetadataFlags( }; } -export function withReplayFailureContext( +function withReplayFailureContext( response: DaemonResponse, action: SessionAction, index: number, @@ -313,29 +256,21 @@ function isReplayArtifactPath(candidate: string): boolean { } } -export function buildReplayActionFlags( - parentFlags: CommandFlags | undefined, - actionFlags: SessionAction['flags'] | undefined, -): CommandFlags { - return mergeParentFlags(parentFlags, { ...(actionFlags ?? {}) }); -} - // fallow-ignore-next-line complexity function actionsContainInterpolation(actions: SessionAction[]): boolean { for (const action of actions) { for (const positional of action.positionals ?? []) { if (typeof positional === 'string' && positional.includes('${')) return true; } - if (action.flags) { - for (const value of Object.values(action.flags)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } - if (action.runtime) { - for (const value of Object.values(action.runtime)) { - if (typeof value === 'string' && value.includes('${')) return true; - } - } + if (containsInterpolation(action.flags)) return true; + if (containsInterpolation(action.runtime)) return true; } return false; } + +function containsInterpolation(value: unknown): boolean { + if (typeof value === 'string') return value.includes('${'); + if (Array.isArray(value)) return value.some(containsInterpolation); + if (value && typeof value === 'object') return Object.values(value).some(containsInterpolation); + return false; +} diff --git a/src/daemon/handlers/session-test-discovery.ts b/src/daemon/handlers/session-test-discovery.ts index df2101840..85ecc5113 100644 --- a/src/daemon/handlers/session-test-discovery.ts +++ b/src/daemon/handlers/session-test-discovery.ts @@ -27,11 +27,13 @@ export function discoverReplayTestEntries(params: { inputs: string[]; cwd?: string; platformFilter?: PlatformSelector; + replayBackend?: string; }): ReplayTestDiscoveryEntry[] { - const { inputs, cwd, platformFilter } = params; + const { inputs, cwd, platformFilter, replayBackend } = params; + const extensions = replayTestExtensions(replayBackend); const resolvedCwd = cwd ?? process.cwd(); const filePaths = [ - ...new Set(inputs.flatMap((input) => expandReplayTestInput(input, resolvedCwd))), + ...new Set(inputs.flatMap((input) => expandReplayTestInput(input, resolvedCwd, extensions))), ] .map((entry) => path.normalize(entry)) .sort((left, right) => left.localeCompare(right)); @@ -45,12 +47,16 @@ export function discoverReplayTestEntries(params: { continue; } if (!metadata.platform) { - entries.push({ - kind: 'skip', - path: filePath, - reason: 'skipped-by-filter', - message: `missing platform metadata for --platform ${platformFilter}`, - }); + if (isMaestroReplayBackend(replayBackend)) { + entries.push({ kind: 'run', path: filePath, metadata }); + } else { + entries.push({ + kind: 'skip', + path: filePath, + reason: 'skipped-by-filter', + message: `missing platform metadata for --platform ${platformFilter}`, + }); + } continue; } if (!matchesPlatformFilter(platformFilter, metadata.platform)) { @@ -62,7 +68,7 @@ export function discoverReplayTestEntries(params: { const runnableCount = entries.filter((entry) => entry.kind === 'run').length; if (runnableCount === 0) { const suffix = platformFilter ? ` for --platform ${platformFilter}` : ''; - throw new AppError('INVALID_ARGS', `No .ad tests matched${suffix}.`); + throw new AppError('INVALID_ARGS', `No replay tests matched${suffix}.`); } return entries; @@ -123,18 +129,20 @@ export function resolveReplayTestRetries( return Math.max(0, Math.min(MAX_REPLAY_TEST_RETRIES, resolved)); } -function expandReplayTestInput(input: string, cwd: string): string[] { +function expandReplayTestInput(input: string, cwd: string, extensions: Set): string[] { const expandedInput = SessionStore.expandHome(input, cwd); if (fs.existsSync(expandedInput)) { const stat = fs.statSync(expandedInput); if (stat.isDirectory()) { - return fs - .globSync('**/*.ad', { cwd: expandedInput }) - .map((match) => path.join(expandedInput, match)); + return replayTestGlobPatterns(extensions).flatMap((pattern) => + fs + .globSync(pattern, { cwd: expandedInput }) + .map((match) => path.join(expandedInput, match)), + ); } if (stat.isFile()) { - if (path.extname(expandedInput) !== '.ad') { - throw new AppError('INVALID_ARGS', `test requires .ad files. Received: ${input}`); + if (!extensions.has(path.extname(expandedInput))) { + throw new AppError('INVALID_ARGS', `test does not support this file type: ${input}`); } return [expandedInput]; } @@ -152,7 +160,21 @@ function expandReplayTestInput(input: string, cwd: string): string[] { return matches .map((match) => (path.isAbsolute(match) ? match : path.resolve(cwd, match))) - .filter((match) => path.extname(match) === '.ad' && isExistingFile(match)); + .filter((match) => extensions.has(path.extname(match)) && isExistingFile(match)); +} + +function replayTestExtensions(replayBackend: string | undefined): Set { + return isMaestroReplayBackend(replayBackend) + ? new Set(['.ad', '.yaml', '.yml']) + : new Set(['.ad']); +} + +function replayTestGlobPatterns(extensions: Set): string[] { + return [...extensions].map((extension) => `**/*${extension}`); +} + +function isMaestroReplayBackend(replayBackend: string | undefined): boolean { + return replayBackend === 'maestro'; } function looksLikeGlob(value: string): boolean { diff --git a/src/daemon/handlers/session-test.ts b/src/daemon/handlers/session-test.ts index 14f21bf9b..35b3d1ab2 100644 --- a/src/daemon/handlers/session-test.ts +++ b/src/daemon/handlers/session-test.ts @@ -41,6 +41,7 @@ export async function runReplayTestSuite( inputs: req.positionals, cwd: req.meta?.cwd, platformFilter: req.flags?.platform, + replayBackend: req.flags?.replayBackend, }); const suiteInvocationId = buildReplayTestInvocationId(req.meta?.requestId); const suiteArtifactsDir = resolveReplayTestArtifactsDir({ diff --git a/src/daemon/handlers/session.ts b/src/daemon/handlers/session.ts index 571c09f0b..08c0c05c3 100644 --- a/src/daemon/handlers/session.ts +++ b/src/daemon/handlers/session.ts @@ -203,13 +203,15 @@ export async function handleSessionCommands(params: { if (req.command === PUBLIC_COMMANDS.keyboard) { const session = sessionStore.get(sessionName); const keyboardAction = req.positionals?.[0]?.trim().toLowerCase(); - if (!session && keyboardAction === 'dismiss') { + const needsForegroundIosApp = + keyboardAction === 'dismiss' || keyboardAction === 'enter' || keyboardAction === 'return'; + if (!session && needsForegroundIosApp) { const flags = req.flags ?? {}; const normalizedPlatform = normalizePlatformSelector(flags.platform); if (normalizedPlatform === 'ios') { return errorResponse( 'SESSION_NOT_FOUND', - 'iOS keyboard dismiss requires an active session so the target app stays foregrounded. Run open first.', + 'iOS keyboard action requires an active session so the target app stays foregrounded. Run open first.', ); } } diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 47ecf8cdb..17b5917ec 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -60,7 +60,10 @@ export async function captureSnapshot(params: CaptureSnapshotParams): Promise<{ androidSnapshot?: AndroidSnapshotBackendMetadata; freshness?: AndroidFreshnessCaptureMeta; }> { - if (params.device.platform === 'ios' && params.session?.postGestureStabilization) { + if ( + (params.device.platform === 'ios' || params.device.platform === 'android') && + params.session?.postGestureStabilization + ) { return { snapshot: await capturePostGestureStabilizedSnapshot({ session: params.session, diff --git a/src/daemon/post-gesture-stabilization.ts b/src/daemon/post-gesture-stabilization.ts index dd4e17dde..a99b400f5 100644 --- a/src/daemon/post-gesture-stabilization.ts +++ b/src/daemon/post-gesture-stabilization.ts @@ -8,7 +8,7 @@ const STABILIZATION_INTERVAL_MS = 200; const RECT_TOLERANCE_PX = 1; export function markPostGestureStabilization(session: SessionState, action: string): void { - if (session.device.platform !== 'ios') return; + if (!supportsPostGestureStabilization(session.device.platform)) return; if (!isPostGestureStabilizingAction(action)) return; session.postGestureStabilization = { action, @@ -27,7 +27,7 @@ export async function capturePostGestureStabilizedSnapshot(params: { }): Promise { const { session, capture } = params; const pending = session?.postGestureStabilization; - if (!session || session.device.platform !== 'ios' || !pending) { + if (!session || !supportsPostGestureStabilization(session.device.platform) || !pending) { return await capture(); } @@ -75,6 +75,10 @@ function isPostGestureStabilizingAction(action: string): boolean { return action === 'swipe' || action === 'scroll'; } +function supportsPostGestureStabilization(platform: SessionState['device']['platform']): boolean { + return platform === 'ios' || platform === 'android'; +} + type StabilityEntry = { key: string; x: number; diff --git a/src/daemon/runtime-session.ts b/src/daemon/runtime-session.ts index 1ef016b54..5ac613d6d 100644 --- a/src/daemon/runtime-session.ts +++ b/src/daemon/runtime-session.ts @@ -7,7 +7,7 @@ export type RuntimeSessionRecordOptions = { metadata?: Record; }; -export function toRuntimeSessionRecord( +function toRuntimeSessionRecord( session: SessionState | undefined, name: string, options: RuntimeSessionRecordOptions = {}, diff --git a/src/daemon/types.ts b/src/daemon/types.ts index d5f3720bd..aa5df7f59 100644 --- a/src/daemon/types.ts +++ b/src/daemon/types.ts @@ -228,6 +228,7 @@ export type SessionAction = { snapshotDepth?: number; snapshotScope?: string; snapshotRaw?: boolean; + launchArgs?: string[]; saveScript?: boolean | string; noRecord?: boolean; }; diff --git a/src/mcp/server.ts b/src/mcp/server.ts index da6da9783..e18fc59d2 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -32,9 +32,7 @@ export async function runAgentDeviceMcpServer(): Promise { }); } -export function handleMcpPayload( - messageOrBatch: JsonRpcMessage | JsonRpcMessage[], -): unknown | null { +function handleMcpPayload(messageOrBatch: JsonRpcMessage | JsonRpcMessage[]): unknown | null { if (Array.isArray(messageOrBatch)) { const responses = messageOrBatch.flatMap((message) => responseArray(handleMcpMessage(message))); return responses.length > 0 ? responses : null; diff --git a/src/platforms/android/__tests__/adb-provider-scope.test.ts b/src/platforms/android/__tests__/adb-provider-scope.test.ts index b1ddec423..50f32347c 100644 --- a/src/platforms/android/__tests__/adb-provider-scope.test.ts +++ b/src/platforms/android/__tests__/adb-provider-scope.test.ts @@ -4,9 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { test } from 'vitest'; import { runCmd } from '../../../utils/exec.ts'; -import { - withAndroidAdbProvider, -} from '../adb-executor.ts'; +import { withAndroidAdbProvider } from '../adb-executor.ts'; const device = { platform: 'android', @@ -65,4 +63,3 @@ test('withAndroidAdbProvider ignores adb commands for another serial', async () assert.equal(result.stdout, 'local -s other-device shell echo local'); assert.deepEqual(calls, []); }); - diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index 375077663..bec00446b 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -734,6 +734,38 @@ test('openAndroidApp reports localhost reverse failures with port context', asyn ); }); +test('openAndroidApp binds deep link URLs to the requested package', async () => { + await withMockedAdb( + 'agent-device-android-open-deep-link-package-', + [ + '#!/bin/sh', + 'printf "__CMD__\\n" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', + 'printf "%s\\n" "$@" >> "$AGENT_DEVICE_TEST_ARGS_FILE"', + 'if [ "$1" = "-s" ]; then', + ' shift', + ' shift', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "pm" ] && [ "$3" = "list" ]; then', + ' echo "package:com.example.app"', + ' exit 0', + 'fi', + 'if [ "$1" = "shell" ] && [ "$2" = "am" ] && [ "$3" = "start" ]; then', + ' echo "Status: ok"', + ' exit 0', + 'fi', + 'exit 0', + '', + ].join('\n'), + async ({ argsLogPath, device }) => { + await openAndroidApp(device, 'com.example.app', { url: 'example://bottom-tabs' }); + const logged = await fs.readFile(argsLogPath, 'utf8'); + assert.match(logged, /shell\nam\nstart\n-W\n-a\nandroid\.intent\.action\.VIEW/); + assert.match(logged, /-d\nexample:\/\/bottom-tabs/); + assert.match(logged, /-p\ncom\.example\.app/); + }, + ); +}); + test('setAndroidSetting appearance toggle flips current mode', async () => { await withMockedAdb( 'agent-device-android-appearance-toggle-', diff --git a/src/platforms/android/__tests__/perf.test.ts b/src/platforms/android/__tests__/perf.test.ts index 46067904b..274f5a726 100644 --- a/src/platforms/android/__tests__/perf.test.ts +++ b/src/platforms/android/__tests__/perf.test.ts @@ -1,9 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; -import { - parseAndroidFramePerfSample, - parseAndroidMemInfoSample, -} from '../perf.ts'; +import { parseAndroidFramePerfSample, parseAndroidMemInfoSample } from '../perf.ts'; test('parseAndroidMemInfoSample supports legacy total row layout', () => { const sample = parseAndroidMemInfoSample( diff --git a/src/platforms/android/__tests__/snapshot-helper.test.ts b/src/platforms/android/__tests__/snapshot-helper.test.ts index 5bd98da4a..45fed04ae 100644 --- a/src/platforms/android/__tests__/snapshot-helper.test.ts +++ b/src/platforms/android/__tests__/snapshot-helper.test.ts @@ -47,6 +47,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () helperApiVersion: '1', outputFormat: 'uiautomator-xml', waitForIdleTimeoutMs: '25', + waitForIdleQuietMs: '10', timeoutMs: '8000', maxDepth: '128', maxNodes: '5000', @@ -66,6 +67,7 @@ test('parseAndroidSnapshotHelperOutput reconstructs XML chunks and metadata', () helperApiVersion: '1', outputFormat: 'uiautomator-xml', waitForIdleTimeoutMs: 25, + waitForIdleQuietMs: 10, timeoutMs: 8000, maxDepth: 128, maxNodes: 5000, @@ -522,6 +524,11 @@ test('ensureAndroidSnapshotHelper retry install also uses provider install capab test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => { let capturedArgs: string[] | undefined; const adb: AndroidAdbExecutor = async (args, options) => { + if (args[0] === 'shell' && args[1] === 'rm') { + assert.equal(options?.allowFailure, true); + assert.equal(options?.timeoutMs, 5000); + return { exitCode: 0, stdout: '', stderr: '' }; + } capturedArgs = args; assert.equal(options?.allowFailure, true); assert.equal(options?.timeoutMs, 14000); @@ -533,6 +540,7 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => ok: 'true', outputFormat: 'uiautomator-xml', waitForIdleTimeoutMs: '10', + waitForIdleQuietMs: '5', timeoutMs: '9000', maxDepth: '64', maxNodes: '100', @@ -545,9 +553,11 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => const result = await captureAndroidSnapshotWithHelper({ adb, waitForIdleTimeoutMs: 10, + waitForIdleQuietMs: 5, timeoutMs: 9000, maxDepth: 64, maxNodes: 100, + outputPath: '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', }); assert.deepEqual(capturedArgs, [ @@ -559,6 +569,9 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => 'waitForIdleTimeoutMs', '10', '-e', + 'waitForIdleQuietMs', + '5', + '-e', 'timeoutMs', '9000', '-e', @@ -567,6 +580,9 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => '-e', 'maxNodes', '100', + '-e', + 'outputPath', + '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', 'com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation', ]); assert.equal(result.xml, ''); @@ -576,7 +592,10 @@ test('captureAndroidSnapshotWithHelper uses injected adb executor', async () => test('captureAndroidSnapshotWithHelper gives adb command overhead beyond helper timeout', async () => { let commandTimeoutMs: number | undefined; await captureAndroidSnapshotWithHelper({ - adb: async (_args, options) => { + adb: async (args, options) => { + if (args[0] === 'shell' && args[1] === 'rm') { + return { exitCode: 0, stdout: '', stderr: '' }; + } commandTimeoutMs = options?.timeoutMs; return { exitCode: 0, @@ -622,6 +641,42 @@ test('captureAndroidSnapshotWithHelper wraps unparseable failed output with adb ); }); +test('captureAndroidSnapshotWithHelper reads helper output file when instrumentation output is unparseable', async () => { + const calls: string[][] = []; + const result = await captureAndroidSnapshotWithHelper({ + adb: async (args) => { + calls.push(args); + if (args[0] === 'shell' && args[1] === 'am') { + return { + exitCode: 0, + stdout: 'INSTRUMENTATION_RESULT: shortMsg=Process crashed.', + stderr: '', + }; + } + if (args[0] === 'shell' && args[1] === 'cat') { + return { + exitCode: 0, + stdout: '', + stderr: '', + }; + } + if (args[0] === 'shell' && args[1] === 'rm') { + return { exitCode: 0, stdout: '', stderr: '' }; + } + throw new Error(`unexpected args: ${args.join(' ')}`); + }, + outputPath: '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', + }); + + assert.equal(result.xml, ''); + assert.equal(result.metadata.outputFormat, 'uiautomator-xml'); + assert.deepEqual(calls.at(1), [ + 'shell', + 'cat', + '/sdcard/Android/data/com.callstack.agentdevice.snapshothelper/files/test.xml', + ]); +}); + test('prepareAndroidSnapshotHelperArtifactFromManifestUrl downloads and verifies APK', async () => { const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'snapshot-helper-download-')); const apk = Buffer.from('downloaded-helper'); diff --git a/src/platforms/android/__tests__/snapshot.test.ts b/src/platforms/android/__tests__/snapshot.test.ts index 6c42d9bee..a592add21 100644 --- a/src/platforms/android/__tests__/snapshot.test.ts +++ b/src/platforms/android/__tests__/snapshot.test.ts @@ -246,6 +246,9 @@ async function withTempScreenshot( function mockAndroidSnapshotXml(xml: string, activityDump = ''): void { mockRunCmd.mockImplementation(async (_cmd, args) => { + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('exec-out')) { return { exitCode: 0, stdout: xml, stderr: '' }; } @@ -256,6 +259,12 @@ function mockAndroidSnapshotXml(xml: string, activityDump = ''): void { }); } +function isAndroidSdkVersionCommand(args: string[]): boolean { + return ( + args.includes('shell') && args.includes('getprop') && args.includes('ro.build.version.sdk') + ); +} + function adbTimeout(args: string[]): AppError { return new AppError('COMMAND_FAILED', 'adb timed out after 8000ms', { cmd: 'adb', @@ -329,7 +338,7 @@ test('snapshotAndroid uses injected helper artifact before stock uiautomator', a assert.equal(result.androidSnapshot.installReason, 'current'); assert.equal(result.androidSnapshot.captureMode, 'interactive-windows'); assert.equal(result.androidSnapshot.windowCount, 1); - assert.deepEqual(timeouts, [30000, 8000]); + assert.deepEqual(timeouts, [30000, 30000]); assert.equal(mockRunCmd.mock.calls.length, 0); }); @@ -410,6 +419,9 @@ test('snapshotAndroid resolves helper adb through scoped provider', async () => stderr: '', }; } + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('instrument')) { return { exitCode: 0, @@ -419,6 +431,9 @@ test('snapshotAndroid resolves helper adb through scoped provider', async () => stderr: '', }; } + if (args[0] === 'shell' && args[1] === 'rm') { + return { exitCode: 0, stdout: '', stderr: '' }; + } throw new Error(`unexpected scoped helper adb args: ${args.join(' ')}`); }, }; @@ -611,6 +626,52 @@ test('snapshotAndroid skips stock fallback after structured helper timeout', asy assert.equal(stockAttempted, false); }); +test('snapshotAndroid skips stock fallback after killed helper instrumentation', async () => { + let stockAttempted = false; + const helperAdb = createHelperAdb({ + instrument: async () => ({ exitCode: 137, stdout: '', stderr: '' }), + stock: async () => { + stockAttempted = true; + throw new Error('stock fallback should not run'); + }, + }); + + await assert.rejects( + () => snapshotAndroidWithHelper(helperAdb), + (error) => { + assert.match( + (error as Error).message, + /Android snapshot helper failed before returning parseable output/, + ); + assert.match((error as Error).message, /Stock UIAutomator fallback was skipped/); + assert.equal((error as { details?: Record }).details?.exitCode, 137); + return true; + }, + ); + assert.equal(stockAttempted, false); +}); + +test('snapshotAndroid skips stock fallback after unparseable helper output', async () => { + let stockAttempted = false; + const helperAdb = createHelperAdb({ + instrument: async () => ({ exitCode: 0, stdout: '', stderr: '' }), + stock: async () => { + stockAttempted = true; + throw new Error('stock fallback should not run'); + }, + }); + + await assert.rejects( + () => snapshotAndroidWithHelper(helperAdb), + (error) => { + assert.match((error as Error).message, /Android snapshot helper output could not be parsed/); + assert.match((error as Error).message, /Stock UIAutomator fallback was skipped/); + return true; + }, + ); + assert.equal(stockAttempted, false); +}); + test('snapshotAndroid falls back to stock dump after helper adb timeout', async () => { const stockXml = ''; @@ -753,6 +814,34 @@ test('dumpUiHierarchy reads fallback XML when dump exits non-zero', async () => assert.equal(catCall?.[2], undefined); }); +test('dumpUiHierarchy does not read a stale fallback file when dump fails without a path', async () => { + mockRunCmd.mockImplementation(async (_cmd, args) => { + if (args.includes('exec-out')) { + return { exitCode: 137, stdout: 'Killed', stderr: '' }; + } + if ( + args.includes('uiautomator') && + args.includes('dump') && + args.includes('/sdcard/window_dump.xml') + ) { + return { exitCode: 137, stdout: 'Killed', stderr: '' }; + } + if (args.includes('cat') && args.includes('/sdcard/window_dump.xml')) { + throw new Error('cat should not read a stale dump file'); + } + throw new Error(`unexpected args: ${args.join(' ')}`); + }); + + await assert.rejects( + dumpUiHierarchy(device), + (error: unknown) => + error instanceof AppError && + error.code === 'COMMAND_FAILED' && + error.message.includes('did not return XML') && + error.details?.reason === 'missing_fresh_dump', + ); +}); + test('dumpUiHierarchy retries when fallback dump file is temporarily missing', async () => { const xml = ''; let catAttempts = 0; @@ -903,6 +992,9 @@ test('snapshotAndroid skips activity dump when snapshot has no scrollable nodes' `; mockRunCmd.mockImplementation(async (_cmd, args) => { + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('exec-out')) { return { exitCode: 0, stdout: xml, stderr: '' }; } @@ -929,6 +1021,9 @@ test('snapshotAndroid skips hidden content hints when disabled', async () => { `; mockRunCmd.mockImplementation(async (_cmd, args) => { + if (isAndroidSdkVersionCommand(args)) { + return { exitCode: 0, stdout: '35', stderr: '' }; + } if (args.includes('exec-out')) { return { exitCode: 0, stdout: xml, stderr: '' }; } diff --git a/src/platforms/android/app-lifecycle.ts b/src/platforms/android/app-lifecycle.ts index fb1ccb118..cc8fdc67f 100644 --- a/src/platforms/android/app-lifecycle.ts +++ b/src/platforms/android/app-lifecycle.ts @@ -291,72 +291,132 @@ async function ensureAndroidLocalhostReverse(device: DeviceInfo, target: string) } } +export type OpenAndroidAppOptions = { + activity?: string; + appBundleId?: string; + url?: string; +}; + export async function openAndroidApp( device: DeviceInfo, app: string, - activity?: string, + optionsOrActivity?: OpenAndroidAppOptions | string, ): Promise { if (!device.booted) { await waitForAndroidBoot(device.id); } + const options = normalizeOpenAndroidAppOptions(optionsOrActivity); + const activity = options.activity; const deepLinkTarget = app.trim(); if (isDeepLinkTarget(deepLinkTarget)) { - if (activity) { - throw new AppError( - 'INVALID_ARGS', - 'Activity override is not supported when opening a deep link URL', - ); - } - await ensureAndroidLocalhostReverse(device, deepLinkTarget); - await runAndroidAdb(device, [ - 'shell', - 'am', - 'start', - '-W', - '-a', - 'android.intent.action.VIEW', - '-d', - deepLinkTarget, - ]); + await openAndroidDeepLink(device, deepLinkTarget, options); + return; + } + if (options.url !== undefined) { + await openAndroidAppBoundDeepLink(device, app, options); return; } const resolved = await resolveAndroidApp(device, app); const launchCategory = resolveAndroidLauncherCategory(device); if (resolved.type === 'intent') { - if (activity) { - throw new AppError( - 'INVALID_ARGS', - 'Activity override requires a package name, not an intent', - ); - } - await runAndroidAdb(device, ['shell', 'am', 'start', '-W', '-a', resolved.value]); + await openAndroidIntent(device, resolved.value, activity); return; } if (activity) { - const component = activity.includes('/') - ? activity - : `${resolved.value}/${activity.startsWith('.') ? activity : `.${activity}`}`; - try { - await runAndroidAdb(device, [ - 'shell', - 'am', - 'start', - '-W', - '-a', - 'android.intent.action.MAIN', - '-c', - ANDROID_DEFAULT_CATEGORY, - '-c', - launchCategory, - '-n', - component, - ]); - } catch (error) { - await maybeRethrowAndroidMissingPackageError(device, resolved.value, error); - throw error; - } + await openAndroidPackageActivity(device, resolved.value, activity, launchCategory); return; } + await openAndroidPackage(device, resolved.value, launchCategory); +} + +async function openAndroidDeepLink( + device: DeviceInfo, + target: string, + options: OpenAndroidAppOptions, +): Promise { + if (options.activity) { + throw new AppError( + 'INVALID_ARGS', + 'Activity override is not supported when opening a deep link URL', + ); + } + await ensureAndroidLocalhostReverse(device, target); + await runAndroidAdb(device, [ + 'shell', + 'am', + 'start', + '-W', + '-a', + 'android.intent.action.VIEW', + '-d', + target, + ...androidDeepLinkPackageArgs(options.appBundleId), + ]); +} + +async function openAndroidAppBoundDeepLink( + device: DeviceInfo, + app: string, + options: OpenAndroidAppOptions, +): Promise { + if (options.activity) { + throw new AppError( + 'INVALID_ARGS', + 'Activity override is not supported when opening an app-bound deep link URL', + ); + } + const deepLinkUrl = options.url?.trim() ?? ''; + if (!isDeepLinkTarget(deepLinkUrl)) { + throw new AppError('INVALID_ARGS', 'Android app-bound open requires a valid URL target'); + } + const resolved = await resolveAndroidPackageForOpen(device, app, 'app-bound open'); + await runAndroidAdb(device, [ + 'shell', + 'am', + 'start', + '-W', + '-a', + 'android.intent.action.VIEW', + '-d', + deepLinkUrl, + '-p', + resolved, + ]); +} + +async function openAndroidIntent( + device: DeviceInfo, + intent: string, + activity: string | undefined, +): Promise { + if (activity) { + throw new AppError('INVALID_ARGS', 'Activity override requires a package name, not an intent'); + } + await runAndroidAdb(device, ['shell', 'am', 'start', '-W', '-a', intent]); +} + +async function openAndroidPackageActivity( + device: DeviceInfo, + packageName: string, + activity: string, + launchCategory: string, +): Promise { + const component = activity.includes('/') + ? activity + : `${packageName}/${activity.startsWith('.') ? activity : `.${activity}`}`; + try { + await runAndroidAdb(device, buildAndroidActivityLaunchArgs(component, launchCategory)); + } catch (error) { + await maybeRethrowAndroidMissingPackageError(device, packageName, error); + throw error; + } +} + +async function openAndroidPackage( + device: DeviceInfo, + packageName: string, + launchCategory: string, +): Promise { const primaryResult = await runAndroidAdb( device, [ @@ -371,24 +431,28 @@ export async function openAndroidApp( '-c', launchCategory, '-p', - resolved.value, + packageName, ], { allowFailure: true }, ); if (primaryResult.exitCode === 0 && !isAmStartError(primaryResult.stdout, primaryResult.stderr)) { return; } - const component = await resolveAndroidLaunchComponent(device, resolved.value); + const component = await resolveAndroidLaunchComponent(device, packageName); if (!component) { - if (!(await isAndroidPackageInstalled(device, resolved.value))) { - throw buildAndroidPackageNotInstalledError(resolved.value); + if (!(await isAndroidPackageInstalled(device, packageName))) { + throw buildAndroidPackageNotInstalledError(packageName); } - throw new AppError('COMMAND_FAILED', `Failed to launch ${resolved.value}`, { + throw new AppError('COMMAND_FAILED', `Failed to launch ${packageName}`, { stdout: primaryResult.stdout, stderr: primaryResult.stderr, }); } - await runAndroidAdb(device, [ + await runAndroidAdb(device, buildAndroidActivityLaunchArgs(component, launchCategory)); +} + +function buildAndroidActivityLaunchArgs(component: string, launchCategory: string): string[] { + return [ 'shell', 'am', 'start', @@ -401,7 +465,31 @@ export async function openAndroidApp( launchCategory, '-n', component, - ]); + ]; +} + +async function resolveAndroidPackageForOpen( + device: DeviceInfo, + app: string, + label: string, +): Promise { + const resolved = await resolveAndroidApp(device, app); + if (resolved.type === 'intent') { + throw new AppError('INVALID_ARGS', `Android ${label} requires a package name, not an intent`); + } + return resolved.value; +} + +function normalizeOpenAndroidAppOptions( + optionsOrActivity: OpenAndroidAppOptions | string | undefined, +): OpenAndroidAppOptions { + if (typeof optionsOrActivity === 'string') return { activity: optionsOrActivity }; + return optionsOrActivity ?? {}; +} + +function androidDeepLinkPackageArgs(packageName: string | undefined): string[] { + const normalized = packageName?.trim(); + return normalized ? ['-p', normalized] : []; } function buildAndroidPackageNotInstalledError(packageName: string): AppError { diff --git a/src/platforms/android/input-actions.ts b/src/platforms/android/input-actions.ts index 45c750521..4a43fd173 100644 --- a/src/platforms/android/input-actions.ts +++ b/src/platforms/android/input-actions.ts @@ -47,6 +47,10 @@ export async function homeAndroid(device: DeviceInfo): Promise { await runAndroidAdb(device, ['shell', 'input', 'keyevent', '3']); } +export async function pressAndroidEnter(device: DeviceInfo): Promise { + await runAndroidAdb(device, ['shell', 'input', 'keyevent', 'ENTER']); +} + export async function rotateAndroid( device: DeviceInfo, orientation: DeviceRotation, diff --git a/src/platforms/android/perf.ts b/src/platforms/android/perf.ts index a7c4bbcc0..8412f2ccd 100644 --- a/src/platforms/android/perf.ts +++ b/src/platforms/android/perf.ts @@ -73,7 +73,7 @@ export async function sampleAndroidMemoryPerf( } } -export function parseAndroidCpuInfoSample( +function parseAndroidCpuInfoSample( stdout: string, packageName: string, measuredAt: string, diff --git a/src/platforms/android/snapshot-helper-capture.ts b/src/platforms/android/snapshot-helper-capture.ts index 4dea0e922..a20ab39c5 100644 --- a/src/platforms/android/snapshot-helper-capture.ts +++ b/src/platforms/android/snapshot-helper-capture.ts @@ -7,6 +7,7 @@ import { ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, ANDROID_SNAPSHOT_HELPER_PACKAGE, ANDROID_SNAPSHOT_HELPER_PROTOCOL, + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS, ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, } from './snapshot-helper-types.ts'; import type { @@ -29,70 +30,172 @@ type AndroidInstrumentationRecordState = { currentResult: Record | null; }; +type AndroidSnapshotHelperResolvedCaptureOptions = { + waitForIdleTimeoutMs: number; + waitForIdleQuietMs: number; + timeoutMs: number; + commandTimeoutMs: number; + maxDepth: number; + maxNodes: number; + packageName: string; + runner: string; + outputPath?: string; +}; + export async function captureAndroidSnapshotWithHelper( options: AndroidSnapshotHelperCaptureOptions, ): Promise { - const waitForIdleTimeoutMs = - options.waitForIdleTimeoutMs ?? ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS; - const timeoutMs = options.timeoutMs ?? 8_000; - const commandTimeoutMs = - options.commandTimeoutMs ?? timeoutMs + ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS; - const maxDepth = options.maxDepth ?? 128; - const maxNodes = options.maxNodes ?? 5_000; - const packageName = options.packageName ?? ANDROID_SNAPSHOT_HELPER_PACKAGE; - const runner = options.instrumentationRunner ?? `${packageName}/.SnapshotInstrumentation`; - const args = [ + const resolved = resolveAndroidSnapshotHelperCaptureOptions(options); + const result = await options.adb(buildAndroidSnapshotHelperArgs(resolved), { + allowFailure: true, + timeoutMs: resolved.commandTimeoutMs, + }); + const output = await readAndroidSnapshotHelperOutput(options, resolved, result); + if (resolved.outputPath) await removeHelperOutputFile(options.adb, resolved.outputPath); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', 'Android snapshot helper failed', { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + helper: output.metadata, + }); + } + return output; +} + +function resolveAndroidSnapshotHelperCaptureOptions( + options: AndroidSnapshotHelperCaptureOptions, +): AndroidSnapshotHelperResolvedCaptureOptions { + const timeoutMs = withDefault(options.timeoutMs, 8_000); + const packageName = withDefault(options.packageName, ANDROID_SNAPSHOT_HELPER_PACKAGE); + return { + waitForIdleTimeoutMs: withDefault( + options.waitForIdleTimeoutMs, + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + ), + waitForIdleQuietMs: withDefault( + options.waitForIdleQuietMs, + ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS, + ), + timeoutMs, + commandTimeoutMs: withDefault( + options.commandTimeoutMs, + timeoutMs + ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS, + ), + maxDepth: withDefault(options.maxDepth, 128), + maxNodes: withDefault(options.maxNodes, 5_000), + packageName, + runner: withDefault(options.instrumentationRunner, `${packageName}/.SnapshotInstrumentation`), + ...(options.outputPath ? { outputPath: options.outputPath } : {}), + }; +} + +function withDefault(value: T | undefined, fallback: T): T { + return value === undefined ? fallback : value; +} + +function buildAndroidSnapshotHelperArgs( + options: AndroidSnapshotHelperResolvedCaptureOptions, +): string[] { + return [ 'shell', 'am', 'instrument', '-w', '-e', 'waitForIdleTimeoutMs', - String(waitForIdleTimeoutMs), + String(options.waitForIdleTimeoutMs), + '-e', + 'waitForIdleQuietMs', + String(options.waitForIdleQuietMs), '-e', 'timeoutMs', - String(timeoutMs), + String(options.timeoutMs), '-e', 'maxDepth', - String(maxDepth), + String(options.maxDepth), '-e', 'maxNodes', - String(maxNodes), - runner, + String(options.maxNodes), + ...(options.outputPath ? ['-e', 'outputPath', options.outputPath] : []), + options.runner, ]; +} - const result = await options.adb(args, { - allowFailure: true, - timeoutMs: commandTimeoutMs, - }); - let output: AndroidSnapshotHelperOutput; +async function readAndroidSnapshotHelperOutput( + options: AndroidSnapshotHelperCaptureOptions, + resolved: AndroidSnapshotHelperResolvedCaptureOptions, + result: Awaited>, +): Promise { try { // The helper can report structured ok=false details even when am exits non-zero. - output = parseAndroidSnapshotHelperOutput(`${result.stdout}\n${result.stderr}`); + return parseAndroidSnapshotHelperOutput(`${result.stdout}\n${result.stderr}`); } catch (error) { - if (error instanceof AppError && result.exitCode !== 0 && error.details?.helper) throw error; - throw new AppError( - 'COMMAND_FAILED', - result.exitCode === 0 - ? 'Android snapshot helper output could not be parsed' - : 'Android snapshot helper failed before returning parseable output', - { - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, - }, - error, - ); + return await readFallbackHelperOutputOrThrow(options, resolved, result, error); } - if (result.exitCode !== 0) { - throw new AppError('COMMAND_FAILED', 'Android snapshot helper failed', { +} + +async function readFallbackHelperOutputOrThrow( + options: AndroidSnapshotHelperCaptureOptions, + resolved: AndroidSnapshotHelperResolvedCaptureOptions, + result: Awaited>, + error: unknown, +): Promise { + if (resolved.outputPath) { + const fileOutput = await readHelperOutputFile(options.adb, resolved.outputPath, { + waitForIdleTimeoutMs: resolved.waitForIdleTimeoutMs, + waitForIdleQuietMs: resolved.waitForIdleQuietMs, + timeoutMs: resolved.timeoutMs, + maxDepth: resolved.maxDepth, + maxNodes: resolved.maxNodes, + }); + if (fileOutput) return fileOutput; + } + if (error instanceof AppError && result.exitCode !== 0 && error.details?.helper) throw error; + throw new AppError( + 'COMMAND_FAILED', + result.exitCode === 0 + ? 'Android snapshot helper output could not be parsed' + : 'Android snapshot helper failed before returning parseable output', + { stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, - helper: output.metadata, - }); - } - return output; + }, + error, + ); +} + +async function readHelperOutputFile( + adb: AndroidSnapshotHelperCaptureOptions['adb'], + outputPath: string, + metadata: Omit, +): Promise { + const result = await adb(['shell', 'cat', outputPath], { + allowFailure: true, + timeoutMs: 5_000, + }); + await removeHelperOutputFile(adb, outputPath); + if (result.exitCode !== 0) return undefined; + const xml = result.stdout.trim(); + if (!xml.includes('')) return undefined; + return { + xml, + metadata: { + ...metadata, + outputFormat: ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, + }, + }; +} + +async function removeHelperOutputFile( + adb: AndroidSnapshotHelperCaptureOptions['adb'], + outputPath: string, +): Promise { + await adb(['shell', 'rm', '-f', outputPath], { + allowFailure: true, + timeoutMs: 5_000, + }); } export function parseAndroidSnapshotHelperOutput(output: string): AndroidSnapshotHelperOutput { @@ -241,6 +344,7 @@ function readHelperMetadata(finalResult: Record): AndroidSnapsho helperApiVersion: finalResult.helperApiVersion, outputFormat: ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT, waitForIdleTimeoutMs: readOptionalNumber(finalResult.waitForIdleTimeoutMs), + waitForIdleQuietMs: readOptionalNumber(finalResult.waitForIdleQuietMs), timeoutMs: readOptionalNumber(finalResult.timeoutMs), maxDepth: readOptionalNumber(finalResult.maxDepth), maxNodes: readOptionalNumber(finalResult.maxNodes), diff --git a/src/platforms/android/snapshot-helper-types.ts b/src/platforms/android/snapshot-helper-types.ts index fede2b5ae..c06430cd6 100644 --- a/src/platforms/android/snapshot-helper-types.ts +++ b/src/platforms/android/snapshot-helper-types.ts @@ -10,6 +10,7 @@ export const ANDROID_SNAPSHOT_HELPER_RUNNER = export const ANDROID_SNAPSHOT_HELPER_PROTOCOL = 'android-snapshot-helper-v1'; export const ANDROID_SNAPSHOT_HELPER_OUTPUT_FORMAT = 'uiautomator-xml'; export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS = 500; +export const ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_QUIET_MS = 100; export const ANDROID_SNAPSHOT_HELPER_COMMAND_OVERHEAD_MS = 5_000; export type { AndroidAdbExecutor } from './adb-executor.ts'; @@ -56,16 +57,19 @@ export type AndroidSnapshotHelperCaptureOptions = { packageName?: string; instrumentationRunner?: string; waitForIdleTimeoutMs?: number; + waitForIdleQuietMs?: number; timeoutMs?: number; commandTimeoutMs?: number; maxDepth?: number; maxNodes?: number; + outputPath?: string; }; export type AndroidSnapshotHelperMetadata = { helperApiVersion?: string; outputFormat: 'uiautomator-xml'; waitForIdleTimeoutMs?: number; + waitForIdleQuietMs?: number; timeoutMs?: number; maxDepth?: number; maxNodes?: number; diff --git a/src/platforms/android/snapshot-types.ts b/src/platforms/android/snapshot-types.ts index 66f072422..809c5f936 100644 --- a/src/platforms/android/snapshot-types.ts +++ b/src/platforms/android/snapshot-types.ts @@ -7,6 +7,7 @@ export type AndroidSnapshotBackendMetadata = { fallbackReason?: string; installReason?: 'missing' | 'outdated' | 'forced' | 'current' | 'skipped'; waitForIdleTimeoutMs?: number; + waitForIdleQuietMs?: number; timeoutMs?: number; maxDepth?: number; maxNodes?: number; diff --git a/src/platforms/android/snapshot.ts b/src/platforms/android/snapshot.ts index d344a71d4..8663444d7 100644 --- a/src/platforms/android/snapshot.ts +++ b/src/platforms/android/snapshot.ts @@ -32,6 +32,8 @@ import { type AndroidAdbExecutor, type AndroidSnapshotHelperArtifact, type AndroidSnapshotHelperInstallPolicy, + type AndroidSnapshotHelperInstallResult, + type AndroidSnapshotHelperOutput, } from './snapshot-helper.ts'; import { ANDROID_SNAPSHOT_MAX_NODES, @@ -41,7 +43,16 @@ import { const UI_HIERARCHY_DUMP_TIMEOUT_MS = 8_000; const HELPER_INSTALL_TIMEOUT_MS = 30_000; const HELPER_CAPTURE_TIMEOUT_MS = 5_000; -const HELPER_COMMAND_TIMEOUT_MS = 8_000; +const HELPER_COMMAND_TIMEOUT_MS = 30_000; +const RETRYABLE_ADB_STDERR_PATTERNS = [ + 'device offline', + 'device not found', + 'transport error', + 'connection reset', + 'broken pipe', + 'timed out', + 'no such file or directory', +] as const; type AndroidSnapshotOptions = SnapshotOptions & { helperArtifact?: AndroidSnapshotHelperArtifact; @@ -107,91 +118,7 @@ async function captureAndroidUiHierarchy( async () => await resolveAndroidSnapshotHelperArtifact(options.helperArtifact), ); if (helper.artifact) { - const helperDeviceKey = getAndroidSnapshotHelperDeviceKey(device); - try { - const adbProvider = resolveAndroidAdbProvider(device, options.helperAdb); - const install = await withDiagnosticTimer( - 'android_snapshot_helper_install', - async () => - await ensureAndroidSnapshotHelper({ - adb, - adbProvider, - artifact: helper.artifact!, - deviceKey: helperDeviceKey, - installPolicy: options.helperInstallPolicy, - timeoutMs: HELPER_INSTALL_TIMEOUT_MS, - }), - { - packageName: helper.artifact.manifest.packageName, - versionCode: helper.artifact.manifest.versionCode, - installPolicy: options.helperInstallPolicy ?? 'missing-or-outdated', - }, - ); - emitDiagnostic({ - phase: 'android_snapshot_helper_install_decision', - data: { - packageName: install.packageName, - versionCode: install.versionCode, - installedVersionCode: install.installedVersionCode, - installed: install.installed, - reason: install.reason, - }, - }); - const capture = await withDiagnosticTimer( - 'android_snapshot_helper_capture', - async () => - await captureAndroidSnapshotWithHelper({ - adb, - packageName: helper.artifact!.manifest.packageName, - instrumentationRunner: helper.artifact!.manifest.instrumentationRunner, - waitForIdleTimeoutMs: - options.helperWaitForIdleTimeoutMs ?? - ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, - commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, - }), - { - packageName: helper.artifact.manifest.packageName, - version: helper.artifact.manifest.version, - timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, - commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, - }, - ); - return { - xml: capture.xml, - metadata: { - backend: 'android-helper', - helperVersion: helper.artifact.manifest.version, - helperApiVersion: capture.metadata.helperApiVersion, - installReason: install.reason, - waitForIdleTimeoutMs: capture.metadata.waitForIdleTimeoutMs, - timeoutMs: capture.metadata.timeoutMs, - maxDepth: capture.metadata.maxDepth, - maxNodes: capture.metadata.maxNodes, - rootPresent: capture.metadata.rootPresent, - captureMode: capture.metadata.captureMode, - windowCount: capture.metadata.windowCount, - nodeCount: capture.metadata.nodeCount, - helperTruncated: capture.metadata.truncated, - elapsedMs: capture.metadata.elapsedMs, - }, - }; - } catch (error) { - const busyError = formatAndroidSnapshotHelperBusyError(error); - if (busyError) throw busyError; - const fallbackReason = formatAndroidSnapshotHelperFallbackReason(error); - emitDiagnostic({ - level: 'warn', - phase: 'android_snapshot_helper_fallback', - data: { reason: fallbackReason }, - }); - forgetAndroidSnapshotHelperInstall({ - deviceKey: helperDeviceKey, - packageName: helper.artifact.manifest.packageName, - versionCode: helper.artifact.manifest.versionCode, - }); - return await captureStockUiHierarchy(device, fallbackReason, adb); - } + return await captureAndroidUiHierarchyWithHelper(device, options, adb, helper.artifact); } emitDiagnostic({ @@ -202,6 +129,147 @@ async function captureAndroidUiHierarchy( return await captureStockUiHierarchy(device, helper.fallbackReason, adb); } +async function captureAndroidUiHierarchyWithHelper( + device: DeviceInfo, + options: AndroidSnapshotOptions, + adb: AndroidAdbExecutor, + artifact: AndroidSnapshotHelperArtifact, +): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { + const helperDeviceKey = getAndroidSnapshotHelperDeviceKey(device); + try { + const install = await installAndroidSnapshotHelper( + device, + options, + adb, + artifact, + helperDeviceKey, + ); + const capture = await captureAndroidUiHierarchyFromHelper(options, adb, artifact); + return formatAndroidHelperCaptureResult(capture, artifact, install.reason); + } catch (error) { + return await recoverAndroidHelperCaptureFailure({ + error, + helperDeviceKey, + artifact, + device, + adb, + }); + } +} + +async function installAndroidSnapshotHelper( + device: DeviceInfo, + options: AndroidSnapshotOptions, + adb: AndroidAdbExecutor, + artifact: AndroidSnapshotHelperArtifact, + deviceKey: string, +): Promise { + const install = await withDiagnosticTimer( + 'android_snapshot_helper_install', + async () => + await ensureAndroidSnapshotHelper({ + adb, + adbProvider: resolveAndroidAdbProvider(device, options.helperAdb), + artifact, + deviceKey, + installPolicy: options.helperInstallPolicy, + timeoutMs: HELPER_INSTALL_TIMEOUT_MS, + }), + { + packageName: artifact.manifest.packageName, + versionCode: artifact.manifest.versionCode, + installPolicy: options.helperInstallPolicy ?? 'missing-or-outdated', + }, + ); + emitDiagnostic({ + phase: 'android_snapshot_helper_install_decision', + data: { + packageName: install.packageName, + versionCode: install.versionCode, + installedVersionCode: install.installedVersionCode, + installed: install.installed, + reason: install.reason, + }, + }); + return install; +} + +async function captureAndroidUiHierarchyFromHelper( + options: AndroidSnapshotOptions, + adb: AndroidAdbExecutor, + artifact: AndroidSnapshotHelperArtifact, +): Promise { + return await withDiagnosticTimer( + 'android_snapshot_helper_capture', + async () => + await captureAndroidSnapshotWithHelper({ + adb, + packageName: artifact.manifest.packageName, + instrumentationRunner: artifact.manifest.instrumentationRunner, + waitForIdleTimeoutMs: + options.helperWaitForIdleTimeoutMs ?? ANDROID_SNAPSHOT_HELPER_WAIT_FOR_IDLE_TIMEOUT_MS, + timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, + commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, + }), + { + packageName: artifact.manifest.packageName, + version: artifact.manifest.version, + timeoutMs: HELPER_CAPTURE_TIMEOUT_MS, + commandTimeoutMs: HELPER_COMMAND_TIMEOUT_MS, + }, + ); +} + +function formatAndroidHelperCaptureResult( + capture: AndroidSnapshotHelperOutput, + artifact: AndroidSnapshotHelperArtifact, + installReason: AndroidSnapshotHelperInstallResult['reason'], +): { xml: string; metadata: AndroidSnapshotBackendMetadata } { + return { + xml: capture.xml, + metadata: { + backend: 'android-helper', + helperVersion: artifact.manifest.version, + helperApiVersion: capture.metadata.helperApiVersion, + installReason, + waitForIdleTimeoutMs: capture.metadata.waitForIdleTimeoutMs, + waitForIdleQuietMs: capture.metadata.waitForIdleQuietMs, + timeoutMs: capture.metadata.timeoutMs, + maxDepth: capture.metadata.maxDepth, + maxNodes: capture.metadata.maxNodes, + rootPresent: capture.metadata.rootPresent, + captureMode: capture.metadata.captureMode, + windowCount: capture.metadata.windowCount, + nodeCount: capture.metadata.nodeCount, + helperTruncated: capture.metadata.truncated, + elapsedMs: capture.metadata.elapsedMs, + }, + }; +} + +async function recoverAndroidHelperCaptureFailure(params: { + error: unknown; + helperDeviceKey: string; + artifact: AndroidSnapshotHelperArtifact; + device: DeviceInfo; + adb: AndroidAdbExecutor; +}): Promise<{ xml: string; metadata: AndroidSnapshotBackendMetadata }> { + const busyError = formatAndroidSnapshotHelperBusyError(params.error); + if (busyError) throw busyError; + const fallbackReason = formatAndroidSnapshotHelperFallbackReason(params.error); + emitDiagnostic({ + level: 'warn', + phase: 'android_snapshot_helper_fallback', + data: { reason: fallbackReason }, + }); + forgetAndroidSnapshotHelperInstall({ + deviceKey: params.helperDeviceKey, + packageName: params.artifact.manifest.packageName, + versionCode: params.artifact.manifest.versionCode, + }); + return await captureStockUiHierarchy(params.device, fallbackReason, params.adb); +} + function formatAndroidSnapshotHelperFallbackReason(error: unknown): string { const normalized = normalizeError(error); const helperMessage = readHelperMessage(normalized.details?.helper); @@ -217,7 +285,13 @@ function formatAndroidSnapshotHelperFallbackReason(error: unknown): string { function formatAndroidSnapshotHelperBusyError(error: unknown): AppError | undefined { const normalized = normalizeError(error); - if (!isStructuredHelperTimeout(normalized.details?.helper, normalized.message)) return undefined; + if ( + !isStructuredHelperTimeout(normalized.details?.helper, normalized.message) && + !isKilledHelperInstrumentationFailure(normalized) && + !isUnsafeStockFallbackHelperReason(normalized.message) + ) { + return undefined; + } const reason = formatAndroidSnapshotHelperFallbackReason(error); const hint = 'Android accessibility snapshots can be blocked by busy or continuously changing app UI. Use screenshot as visual truth after this timeout and report the busy UI if it persists.'; @@ -232,6 +306,20 @@ function formatAndroidSnapshotHelperBusyError(error: unknown): AppError | undefi ); } +function isKilledHelperInstrumentationFailure(error: { + message: string; + details?: Record; +}): boolean { + if (error.details?.exitCode !== 137) return false; + return /Android snapshot helper (failed before returning parseable output|output could not be parsed)/.test( + error.message, + ); +} + +function isUnsafeStockFallbackHelperReason(reason: string): boolean { + return /Android snapshot helper output could not be parsed/.test(reason); +} + function readHelperMessage(helper: unknown): string | undefined { if (!helper || typeof helper !== 'object' || !('message' in helper)) return undefined; const message = String(helper.message).trim(); @@ -328,8 +416,6 @@ function enrichStockSnapshotFailureWithHelperReason( } function getAndroidSnapshotHelperDeviceKey(device: DeviceInfo): string { - // Emulator serials are port-based and can be reused after restart; capture failure invalidates - // this key before falling back so stale process-local entries self-heal on the next snapshot. return `${device.platform}:${device.id}`; } @@ -390,7 +476,16 @@ async function dumpUiHierarchyOnce(adb: AndroidAdbExecutor): Promise { allowFailure: true, timeoutMs: UI_HIERARCHY_DUMP_TIMEOUT_MS, }); - const actualPath = resolveDumpPath(dumpPath, dumpResult.stdout, dumpResult.stderr); + const reportedPath = readDumpPath(dumpResult.stdout, dumpResult.stderr); + if (dumpResult.exitCode !== 0 && !reportedPath) { + throw new AppError('COMMAND_FAILED', 'uiautomator dump did not return XML', { + stdout: dumpResult.stdout, + stderr: dumpResult.stderr, + exitCode: dumpResult.exitCode, + reason: 'missing_fresh_dump', + }); + } + const actualPath = reportedPath ?? dumpPath; const result = await adb(['shell', 'cat', actualPath]); const xml = extractUiDumpXml(result.stdout, result.stderr); @@ -403,10 +498,10 @@ async function dumpUiHierarchyOnce(adb: AndroidAdbExecutor): Promise { return xml; } -function resolveDumpPath(defaultPath: string, stdout: string, stderr: string): string { +function readDumpPath(stdout: string, stderr: string): string | undefined { const text = `${stdout}\n${stderr}`; const match = /dumped to:\s*(\S+)/i.exec(text); - return match?.[1] ?? defaultPath; + return match?.[1]; } function extractUiDumpXml(stdout: string, stderr: string): string | null { @@ -425,14 +520,7 @@ function isRetryableAdbError(err: unknown): boolean { if (err.code !== 'COMMAND_FAILED') return false; const rawStderr = err.details?.stderr; const stderr = (typeof rawStderr === 'string' ? rawStderr : '').toLowerCase(); - if (stderr.includes('device offline')) return true; - if (stderr.includes('device not found')) return true; - if (stderr.includes('transport error')) return true; - if (stderr.includes('connection reset')) return true; - if (stderr.includes('broken pipe')) return true; - if (stderr.includes('timed out')) return true; - if (stderr.includes('no such file or directory')) return true; - return false; + return RETRYABLE_ADB_STDERR_PATTERNS.some((pattern) => stderr.includes(pattern)); } function isUiHierarchyDumpTimeout(err: unknown): err is AppError { @@ -440,14 +528,16 @@ function isUiHierarchyDumpTimeout(err: unknown): err is AppError { if (err.code !== 'COMMAND_FAILED') return false; const timeoutMs = err.details?.timeoutMs; if (typeof timeoutMs !== 'number') return false; - const cmd = err.details?.cmd; - const rawArgs = err.details?.args; + return err.details?.cmd === 'adb' && isUiAutomatorDumpArgs(err.details?.args); +} + +function isUiAutomatorDumpArgs(rawArgs: unknown): boolean { const args = Array.isArray(rawArgs) ? rawArgs.map(String) : typeof rawArgs === 'string' ? rawArgs.split(/\s+/) : []; - return cmd === 'adb' && args.includes('uiautomator') && args.includes('dump'); + return args.includes('uiautomator') && args.includes('dump'); } async function dumpActivityTop( diff --git a/src/platforms/ios/__tests__/runner-client.test.ts b/src/platforms/ios/__tests__/runner-client.test.ts index 424179bae..ff12c6da4 100644 --- a/src/platforms/ios/__tests__/runner-client.test.ts +++ b/src/platforms/ios/__tests__/runner-client.test.ts @@ -31,6 +31,7 @@ vi.mock('../runner-macos-products.ts', async () => { import type { DeviceInfo } from '../../../utils/device.ts'; import { AppError } from '../../../utils/errors.ts'; +import type { RunnerCommand } from '../runner-contract.ts'; import { assertSafeDerivedCleanup, isRetryableRunnerError, @@ -98,6 +99,70 @@ const macOsDevice: DeviceInfo = { booted: true, }; +const runnerProtocolCommandFixtures: Record = { + tap: { command: 'tap', x: 120, y: 240 }, + mouseClick: { command: 'mouseClick', x: 120, y: 240, button: 'secondary' }, + tapSeries: { command: 'tapSeries', x: 120, y: 240, count: 2, intervalMs: 80 }, + longPress: { command: 'longPress', x: 120, y: 240, durationMs: 750 }, + interactionFrame: { command: 'interactionFrame' }, + drag: { command: 'drag', x: 120, y: 240, x2: 300, y2: 420, durationMs: 400 }, + dragSeries: { + command: 'dragSeries', + x: 120, + y: 240, + x2: 300, + y2: 420, + count: 2, + pauseMs: 100, + pattern: 'ping-pong', + }, + remotePress: { command: 'remotePress', remoteButton: 'down', durationMs: 250 }, + type: { command: 'type', text: 'hello', delayMs: 20, textEntryMode: 'replace' }, + swipe: { command: 'swipe', direction: 'down', durationMs: 250 }, + findText: { command: 'findText', text: 'Settings' }, + querySelector: { command: 'querySelector', selectorKey: 'id', selectorValue: 'submit' }, + readText: { command: 'readText' }, + snapshot: { + command: 'snapshot', + interactiveOnly: true, + compact: true, + depth: 2, + scope: 'app', + raw: false, + }, + screenshot: { command: 'screenshot', outPath: '/tmp/runner-screenshot.png', fullscreen: true }, + back: { command: 'back' }, + backInApp: { command: 'backInApp' }, + backSystem: { command: 'backSystem' }, + home: { command: 'home' }, + rotate: { command: 'rotate', orientation: 'landscape-left' }, + appSwitcher: { command: 'appSwitcher' }, + keyboardDismiss: { command: 'keyboardDismiss' }, + keyboardReturn: { command: 'keyboardReturn' }, + alert: { command: 'alert', action: 'accept' }, + pinch: { command: 'pinch', scale: 0.5 }, + rotateGesture: { command: 'rotateGesture', degrees: 35, x: 200, y: 420, velocity: 1 }, + transformGesture: { + command: 'transformGesture', + x: 200, + y: 420, + dx: 80, + dy: -40, + scale: 2, + degrees: 35, + durationMs: 700, + }, + recordStart: { + command: 'recordStart', + outPath: '/tmp/runner-recording.mp4', + fps: 30, + quality: 7, + }, + recordStop: { command: 'recordStop' }, + uptime: { command: 'uptime' }, + shutdown: { command: 'shutdown' }, +}; + const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../../../'); async function makeTmpDir(): Promise { @@ -266,6 +331,55 @@ test('resolveRunnerDestination uses simulator destination for simulators', () => assert.equal(resolveRunnerDestination(iosSimulator), 'platform=iOS Simulator,id=sim-1'); }); +test('runner protocol fixtures cover every runner command with JSON-safe samples', () => { + const commands = Object.keys(runnerProtocolCommandFixtures).sort(); + assert.deepEqual(commands, [ + 'alert', + 'appSwitcher', + 'back', + 'backInApp', + 'backSystem', + 'drag', + 'dragSeries', + 'findText', + 'home', + 'interactionFrame', + 'keyboardDismiss', + 'keyboardReturn', + 'longPress', + 'mouseClick', + 'pinch', + 'querySelector', + 'readText', + 'recordStart', + 'recordStop', + 'remotePress', + 'rotate', + 'rotateGesture', + 'screenshot', + 'shutdown', + 'snapshot', + 'swipe', + 'tap', + 'tapSeries', + 'transformGesture', + 'type', + 'uptime', + ]); + + const roundTrip = JSON.parse(JSON.stringify(runnerProtocolCommandFixtures)) as Record< + string, + Record + >; + assert.equal(roundTrip.tap.command, 'tap'); + assert.equal(roundTrip.mouseClick.button, 'secondary'); + assert.equal(roundTrip.snapshot.scope, 'app'); + assert.equal(roundTrip.screenshot.fullscreen, true); + assert.equal(roundTrip.rotate.orientation, 'landscape-left'); + assert.equal(roundTrip.recordStart.fps, 30); + assert.equal(roundTrip.recordStart.quality, 7); +}); + test('resolveRunnerDestination uses device destination for physical devices', () => { assert.equal(resolveRunnerDestination(iosDevice), 'platform=iOS,id=00008110-000E12341234002E'); }); diff --git a/src/platforms/ios/apps.ts b/src/platforms/ios/apps.ts index 70bd66553..ee8ce536c 100644 --- a/src/platforms/ios/apps.ts +++ b/src/platforms/ios/apps.ts @@ -127,7 +127,7 @@ export async function resolveIosApp(device: DeviceInfo, app: string): Promise { const launchConsole = options?.launchConsole?.trim(); if (launchConsole && (device.platform !== 'ios' || device.kind !== 'simulator')) { @@ -185,7 +185,10 @@ export async function openIosApp( const bundleId = options?.appBundleId ?? (await resolveIosApp(device, app)); if (device.kind === 'simulator') { - await launchIosSimulatorApp(device, bundleId, launchConsole ? { launchConsole } : undefined); + await launchIosSimulatorApp(device, bundleId, { + ...(launchConsole ? { launchConsole } : {}), + ...(options?.launchArgs ? { launchArgs: options.launchArgs } : {}), + }); return; } @@ -235,6 +238,53 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { + if (device.platform !== 'ios' || device.kind !== 'simulator') { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Clearing app state is currently supported only on iOS simulators.', + ); + } + + const bundleId = await resolveIosApp(device, app); + await ensureBootedSimulator(device); + await closeIosApp(device, bundleId); + + const result = await runSimctl(device, ['get_app_container', device.id, bundleId, 'data'], { + allowFailure: true, + }); + if (result.exitCode !== 0) { + throw new AppError('COMMAND_FAILED', `simctl get_app_container failed for ${bundleId}`, { + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }); + } + + const containerPath = result.stdout.trim(); + if (!containerPath) { + throw new AppError( + 'COMMAND_FAILED', + `simctl get_app_container returned an empty data container path for ${bundleId}`, + ); + } + + const entries = await fs.readdir(containerPath); + await Promise.all( + entries.map((entry) => + fs.rm(path.join(containerPath, entry), { + recursive: true, + force: true, + }), + ), + ); + + return { bundleId, containerPath }; +} + export async function uninstallIosApp( device: DeviceInfo, app: string, @@ -884,7 +934,7 @@ function isIosBiometricCapabilityMissing(stdout: string, stderr: string): boolea async function launchIosSimulatorApp( device: DeviceInfo, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): Promise { await ensureBootedSimulator(device); @@ -947,11 +997,12 @@ async function launchIosSimulatorApp( function buildIosSimulatorLaunchArgs( deviceId: string, bundleId: string, - options?: { launchConsole?: string }, + options?: { launchConsole?: string; launchArgs?: string[] }, ): string[] { const args = ['launch']; if (options?.launchConsole) args.push('--console-pty'); args.push(deviceId, bundleId); + if (options?.launchArgs) args.push(...options.launchArgs); return args; } diff --git a/src/platforms/ios/interactions.ts b/src/platforms/ios/interactions.ts index 50f91f0df..4ec397caf 100644 --- a/src/platforms/ios/interactions.ts +++ b/src/platforms/ios/interactions.ts @@ -39,6 +39,7 @@ type IosRunnerOverrides = Pick< | 'longPress' | 'focus' | 'type' + | 'fillElementSelector' | 'fill' | 'scroll' | 'pinch' @@ -81,6 +82,7 @@ export function iosRunnerOverrides( command: 'tap', selectorKey: selector.key, selectorValue: selector.value, + allowNonHittableCoordinateFallback: selector.allowNonHittableCoordinateFallback, appBundleId: ctx.appBundleId, }, runnerOpts, @@ -159,7 +161,23 @@ export function iosRunnerOverrides( command: 'type', text, delayMs, - textEntryMode: 'append', + textEntryMode: text === '\n' ? undefined : 'append', + appBundleId: ctx.appBundleId, + }, + runnerOpts, + ); + }, + fillElementSelector: async (selector, text, delayMs) => { + return await runIosRunnerCommand( + device, + { + command: 'type', + selectorKey: selector.key, + selectorValue: selector.value, + allowNonHittableCoordinateFallback: selector.allowNonHittableCoordinateFallback, + text, + delayMs, + textEntryMode: 'replace', appBundleId: ctx.appBundleId, }, runnerOpts, diff --git a/src/platforms/ios/runner-client.ts b/src/platforms/ios/runner-client.ts index 71506ec6f..3ddb7c6b6 100644 --- a/src/platforms/ios/runner-client.ts +++ b/src/platforms/ios/runner-client.ts @@ -28,7 +28,6 @@ import { } from './runner-provider.ts'; import { ensureXctestrun } from './runner-xctestrun.ts'; export { - isReadOnlyRunnerCommand, isRetryableRunnerError, resolveRunnerEarlyExitHint, resolveRunnerBuildFailureHint, diff --git a/src/platforms/ios/runner-contract.ts b/src/platforms/ios/runner-contract.ts index 598ed94f2..c7958a099 100644 --- a/src/platforms/ios/runner-contract.ts +++ b/src/platforms/ios/runner-contract.ts @@ -34,6 +34,7 @@ export type RunnerCommand = { | 'transformGesture' | 'appSwitcher' | 'keyboardDismiss' + | 'keyboardReturn' | 'alert' | 'pinch' | 'recordStart' @@ -44,6 +45,7 @@ export type RunnerCommand = { text?: string; selectorKey?: 'id' | 'label' | 'text' | 'value'; selectorValue?: string; + allowNonHittableCoordinateFallback?: boolean; delayMs?: number; textEntryMode?: 'append' | 'replace'; action?: 'get' | 'accept' | 'dismiss'; diff --git a/src/replay/__tests__/script.test.ts b/src/replay/__tests__/script.test.ts index dba907b43..ac72b639b 100644 --- a/src/replay/__tests__/script.test.ts +++ b/src/replay/__tests__/script.test.ts @@ -290,4 +290,3 @@ test('readReplayScriptMetadata rejects conflicting metadata keys in context head /Conflicting replay test metadata "timeoutMs"/.test(error.message), ); }); - diff --git a/src/replay/vars.ts b/src/replay/vars.ts index dc3fb7d54..7f1ce516c 100644 --- a/src/replay/vars.ts +++ b/src/replay/vars.ts @@ -2,7 +2,7 @@ import { AppError } from '../utils/errors.ts'; import type { SessionAction } from '../daemon/types.ts'; export type ReplayVarScope = { - readonly values: Readonly>; + values: Readonly>; }; export type ReplayVarSources = { @@ -13,7 +13,7 @@ export type ReplayVarSources = { }; export const REPLAY_VAR_KEY_RE = /^[A-Z_][A-Z0-9_]*$/; -const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_]*)(?::-((?:[^}\\]|\\.)*))?\}/g; +const INTERPOLATION_RE = /(\\\$\{)|\$\{([A-Za-z_][A-Za-z0-9_.]*)(?::-((?:[^}\\]|\\.)*))?\}/g; const SHELL_PREFIX = 'AD_VAR_'; const RESERVED_NAMESPACE_PREFIX = 'AD_'; @@ -53,6 +53,13 @@ export function buildReplayVarScope(sources: ReplayVarSources): ReplayVarScope { return { values: merged }; } +export function mergeReplayVarScopeValues( + scope: ReplayVarScope, + values: Record, +): void { + Object.assign(scope.values as Record, values); +} + export function collectReplayShellEnv(processEnv: NodeJS.ProcessEnv): Record { const result: Record = {}; for (const [rawKey, value] of Object.entries(processEnv)) { @@ -156,11 +163,20 @@ function resolveStringProps( loc: { file: string; line: number }, ): T | undefined { if (!obj) return obj; - const next: Record = { ...(obj as Record) }; - for (const [key, value] of Object.entries(next)) { - if (typeof value === 'string') { - next[key] = resolveReplayString(value, scope, loc); - } + return resolveStringValue(obj, scope, loc) as T; +} + +function resolveStringValue( + value: unknown, + scope: ReplayVarScope, + loc: { file: string; line: number }, +): unknown { + if (typeof value === 'string') return resolveReplayString(value, scope, loc); + if (Array.isArray(value)) return value.map((entry) => resolveStringValue(entry, scope, loc)); + if (value && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value).map(([key, entry]) => [key, resolveStringValue(entry, scope, loc)]), + ); } - return next as T; + return value; } diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index a42b09922..4792ae804 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -120,13 +120,34 @@ test('parseArgs recognizes command-specific flag combinations', async () => { }, { label: 'replay maestro flow', - argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada'], + argv: ['replay', './flow.yaml', '--maestro', '--env', 'USER=Ada', '--timeout', '240000'], strictFlags: true, assertParsed: (parsed) => { assert.equal(parsed.command, 'replay'); assert.deepEqual(parsed.positionals, ['./flow.yaml']); assert.equal(parsed.flags.replayMaestro, true); assert.deepEqual(parsed.flags.replayEnv, ['USER=Ada']); + assert.equal(parsed.flags.timeoutMs, 240000); + }, + }, + { + label: 'test maestro suite', + argv: [ + 'test', + './e2e/maestro', + '--maestro', + '--env', + 'APP_ID=com.example', + '--platform', + 'android', + ], + strictFlags: true, + assertParsed: (parsed) => { + assert.equal(parsed.command, 'test'); + assert.deepEqual(parsed.positionals, ['./e2e/maestro']); + assert.equal(parsed.flags.replayMaestro, true); + assert.deepEqual(parsed.flags.replayEnv, ['APP_ID=com.example']); + assert.equal(parsed.flags.platform, 'android'); }, }, ]; @@ -369,6 +390,10 @@ test('parseArgs accepts keyboard subcommands', () => { const dismiss = parseArgs(['keyboard', 'dismiss'], { strictFlags: true }); assert.equal(dismiss.command, 'keyboard'); assert.deepEqual(dismiss.positionals, ['dismiss']); + + const enter = parseArgs(['keyboard', 'enter'], { strictFlags: true }); + assert.equal(enter.command, 'keyboard'); + assert.deepEqual(enter.positionals, ['enter']); }); test('parseArgs accepts scroll pixel distance flag', () => { @@ -917,14 +942,22 @@ test('usageForCommand includes Maestro replay flag', () => { assert.match(help, /--maestro/); assert.match(help, /doubleTapOn/); assert.match(help, /pasteText/); - assert.match(help, /setPermissions/); - assert.match(help, /startRecording\/stopRecording/); assert.match(help, /runFlow file\/inline/); + assert.match(help, /ordered trusted runScript/); assert.match(help, /repeat\.times/); + assert.match(help, /stopApp/); assert.match(help, /Unsupported syntax fails loudly/); assert.match(help, /issues\/558/); }); +test('usageForCommand includes Maestro test suite flag', () => { + const help = usageForCommand('test'); + if (help === null) throw new Error('Expected test help text'); + assert.match(help, /Run one or more replay scripts as a serial test suite/); + assert.match(help, /--maestro/); + assert.match(help, /Replay\/Test: inject or override/); +}); + test('usageForCommand resolves workflow help topic', () => { const help = usageForCommand('workflow'); if (help === null) throw new Error('Expected workflow help text'); @@ -1318,7 +1351,7 @@ test('usage renders concise commands inline with descriptions', () => { / metro prepare --public-base-url \| --proxy-base-url ; metro reload\s{2,}Prepare Metro or reload apps/, ); assert.match(help, / batch --steps \| --steps-file \s{2,}Run multiple commands/); - assert.match(help, / test \.\.\.\s{2,}Run \.ad test suites/); + assert.match(help, / test \.\.\.\s{2,}Run replay test suites/); assert.match(help, / session list\s{2,}List active sessions/); assert.doesNotMatch(help, / metro prepare[^\n]*--project-root/); assert.doesNotMatch(help, /\n batch\s{2,}Run multiple commands/); @@ -1329,7 +1362,8 @@ test('command usage describes test suite flags', () => { const help = usageForCommand('test'); if (help === null) throw new Error('Expected command help text'); assert.match(help, /Usage:\s+agent-device test \.\.\./); - assert.match(help, /Run one or more \.ad scripts as a serial test suite/); + assert.match(help, /Run one or more replay scripts as a serial test suite/); + assert.match(help, /--maestro/); assert.match(help, /--fail-fast/); assert.match(help, /--timeout /); assert.match(help, /--retries /); @@ -1425,8 +1459,11 @@ test('clipboard command usage is documented', () => { test('keyboard command usage is documented', () => { const help = usageForCommand('keyboard'); if (help === null) throw new Error('Expected command help text'); - assert.match(help, /keyboard \[status\|get\|dismiss\]/); - assert.match(help, /Inspect Android keyboard visibility\/type or dismiss the device keyboard/); + assert.match(help, /keyboard \[status\|get\|dismiss\|enter\|return\]/); + assert.match( + help, + /Inspect Android keyboard visibility\/type or press\/dismiss the device keyboard/, + ); }); test('rotate command usage is documented', () => { diff --git a/src/utils/__tests__/daemon-client.test.ts b/src/utils/__tests__/daemon-client.test.ts index 9f65f1b74..991544433 100644 --- a/src/utils/__tests__/daemon-client.test.ts +++ b/src/utils/__tests__/daemon-client.test.ts @@ -13,6 +13,7 @@ import { } from '../../__tests__/test-utils/index.ts'; import { runCmdBackground } from '../exec.ts'; import { + canConnectSocket, cleanupFailedDaemonStartupMetadata, computeDaemonCodeSignature, downloadRemoteArtifact, @@ -285,6 +286,40 @@ test('cleanupFailedDaemonStartupMetadata removes stale daemon metadata on timeou } }); +test('canConnectSocket times out stalled local daemon probes', async () => { + const originalCreateConnection = net.createConnection; + let timeoutMs: number | undefined; + let destroyed = false; + const socket = new EventEmitter() as EventEmitter & { + destroy: () => void; + setTimeout: (ms: number) => typeof socket; + }; + socket.destroy = () => { + destroyed = true; + }; + socket.setTimeout = (ms: number) => { + timeoutMs = ms; + setImmediate(() => socket.emit('timeout')); + return socket; + }; + + (net as unknown as { createConnection: typeof net.createConnection }).createConnection = (( + _options: unknown, + _listener?: () => void, + ) => socket) as typeof net.createConnection; + + try { + const reachable = await canConnectSocket(65_530); + + assert.equal(reachable, false); + assert.equal(timeoutMs, 500); + assert.equal(destroyed, true); + } finally { + (net as unknown as { createConnection: typeof net.createConnection }).createConnection = + originalCreateConnection; + } +}); + test('sendToDaemon reuses reachable local socket daemon metadata', async (t) => { if (!(await supportsLoopbackBind())) { t.skip('loopback listeners are not permitted in this environment'); diff --git a/src/utils/__tests__/device.test.ts b/src/utils/__tests__/device.test.ts index b66eb5ad1..0426e52c8 100644 --- a/src/utils/__tests__/device.test.ts +++ b/src/utils/__tests__/device.test.ts @@ -158,4 +158,3 @@ test('resolveDevice returns physical device when explicitly selected by deviceNa }); assert.equal(result.id, 'phys-1'); }); - diff --git a/src/utils/__tests__/selector-is-predicates.test.ts b/src/utils/__tests__/selector-is-predicates.test.ts new file mode 100644 index 000000000..4e3cb9730 --- /dev/null +++ b/src/utils/__tests__/selector-is-predicates.test.ts @@ -0,0 +1,98 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { evaluateIsPredicate } from '../selector-is-predicates.ts'; +import type { SnapshotNode } from '../snapshot.ts'; + +test('visible predicate treats zero-height hittable Android nodes as hidden', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + ref: 'e1', + parentIndex: 0, + type: 'android.widget.Button', + identifier: 'tab-4', + label: 'Tab 4', + rect: { x: 0, y: 800, width: 100, height: 0 }, + hittable: true, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[1]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, false); +}); + +test('visible predicate treats rectless hittable Android nodes as hidden', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + ref: 'e1', + type: 'android.widget.Button', + label: 'Library', + hittable: true, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[1]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, false); +}); + +test('visible predicate uses visible Android ancestor geometry for rectless text', () => { + const nodes: SnapshotNode[] = [ + { + index: 0, + ref: 'e0', + type: 'android.widget.FrameLayout', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + ref: 'e1', + parentIndex: 0, + type: 'android.widget.Button', + label: 'Library', + rect: { x: 20, y: 100, width: 160, height: 80 }, + hittable: true, + }, + { + index: 2, + ref: 'e2', + parentIndex: 1, + type: 'android.widget.TextView', + label: 'Library', + hittable: false, + }, + ]; + + const result = evaluateIsPredicate({ + predicate: 'visible', + node: nodes[2]!, + nodes, + platform: 'android', + }); + + assert.equal(result.pass, true); +}); diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index 2480965bb..f94fbd3c1 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -358,9 +358,11 @@ React Native dev loop: There is no open-url command; use open with the URL target or host + URL form. Direct iOS URL open remains valid when no host shell is known, but verify that the app UI loaded: agent-device open exp://127.0.0.1:8081 --platform ios - Android uses the URL target directly; do not write open there: + Android Expo URLs can be opened directly when no specific app package must be forced: agent-device open exp://127.0.0.1:8081 --platform android - Android URL/deep-link opens infer the foreground package after launch when possible, so logs/perf can remain package-bound. If perf still says no package is associated, open the host package/app id first, then open the URL in the same session. + Android app-bound deep links can use open when a known package must handle the link: + agent-device open com.example.app example://screen --platform android + Android URL/deep-link opens infer the foreground package after launch when possible, so logs/perf can remain package-bound. If perf still says no package is associated, use the app-bound form when the package id is known. If apps lookup misses the project but shows Expo Go/dev-client and a project URL is available, open the URL/host shell; if no URL is available, ask instead of inventing an app id. Expo Dev Client/development builds: open the installed dev-client app id/name; if a dev-client URL is provided, open that URL next. For Metro setup use metro prepare --kind expo. @@ -1263,7 +1265,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'boolean', usageLabel: '--maestro', usageDescription: - 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp without state-reset side effects, runFlow file/inline with when.platform, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn, doubleTapOn, longPressOn, inputText, pasteText, openLink, assertVisible, assertNotVisible, assertTrue literal true/false, extendedWaitUntil, scroll, absolute/percentage swipe, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, stopApp/killApp, setAirplaneMode, setLocation, setOrientation, supported setPermissions targets, and startRecording/stopRecording. ' + + 'Replay: treat input as a Maestro YAML compatibility flow. Supported subset: launchApp with Apple-platform launch arguments and iOS simulator clearState, runFlow file/inline with when.platform/visible/notVisible/true, ordered trusted runScript file/env steps with http.post/json/output variables, onFlowStart/onFlowComplete, deterministic repeat.times, tapOn including optional, index, childOf, label, and absolute/percentage point taps, doubleTapOn, longPressOn, inputText, eraseText for focused fields, pasteText, openLink, assertVisible, assertNotVisible, extendedWaitUntil, scroll, scrollUntilVisible, absolute/percentage swipe plus swipe.label, takeScreenshot, hideKeyboard, pressKey back/enter/home, back, waitForAnimationToEnd, and stopApp. ' + 'Unsupported syntax fails loudly with a link to https://github.com/callstackincubator/agent-device/issues/558', }, { @@ -1288,7 +1290,7 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ type: 'int', min: 1, usageLabel: '--timeout ', - usageDescription: 'Test: maximum wall-clock time per script attempt', + usageDescription: 'Replay/Test: maximum wall-clock time per script attempt', }, { key: 'retries', @@ -1599,9 +1601,10 @@ const COMMAND_SCHEMAS: Record = { allowedFlags: [], }, keyboard: { - usageOverride: 'keyboard [status|get|dismiss]', - helpDescription: 'Inspect Android keyboard visibility/type or dismiss the device keyboard', - summary: 'Inspect or dismiss the device keyboard', + usageOverride: 'keyboard [status|get|dismiss|enter|return]', + helpDescription: + 'Inspect Android keyboard visibility/type or press/dismiss the device keyboard', + summary: 'Inspect, press, or dismiss the device keyboard', positionalArgs: ['action?'], allowedFlags: [], }, @@ -1676,18 +1679,19 @@ const COMMAND_SCHEMAS: Record = { replay: { helpDescription: 'Replay a recorded session', positionalArgs: ['path'], - allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv'], + allowedFlags: ['replayUpdate', 'replayMaestro', 'replayEnv', 'timeoutMs'], skipCapabilityCheck: true, }, test: { usageOverride: 'test ...', listUsageOverride: 'test ...', - helpDescription: 'Run one or more .ad scripts as a serial test suite', - summary: 'Run .ad test suites', + helpDescription: 'Run one or more replay scripts as a serial test suite', + summary: 'Run replay test suites', positionalArgs: ['pathOrGlob'], allowsExtraPositionals: true, allowedFlags: [ 'replayUpdate', + 'replayMaestro', 'replayEnv', 'failFast', 'timeoutMs', diff --git a/src/utils/keyboard-actions.ts b/src/utils/keyboard-actions.ts new file mode 100644 index 000000000..92fa204e6 --- /dev/null +++ b/src/utils/keyboard-actions.ts @@ -0,0 +1,7 @@ +const KEYBOARD_ACTIONS = ['status', 'get', 'dismiss', 'enter', 'return'] as const; + +export type KeyboardAction = (typeof KEYBOARD_ACTIONS)[number]; + +export function isKeyboardAction(action: string): action is KeyboardAction { + return KEYBOARD_ACTIONS.includes(action as KeyboardAction); +} diff --git a/src/utils/selector-is-predicates.ts b/src/utils/selector-is-predicates.ts index 9c369f548..0d94a6ad9 100644 --- a/src/utils/selector-is-predicates.ts +++ b/src/utils/selector-is-predicates.ts @@ -21,7 +21,8 @@ export function evaluateIsPredicate(params: { const actualText = extractNodeText(node); const editable = isNodeEditable(node, platform); const selected = node.selected === true; - const visible = predicate === 'text' ? isNodeVisible(node) : isAssertionVisible(node, nodes); + const visible = + predicate === 'text' ? isNodeVisible(node) : isAssertionVisible(node, nodes, platform); let pass = false; switch (predicate) { case 'visible': @@ -54,14 +55,14 @@ export function evaluateIsPredicate(params: { function isAssertionVisible( node: SnapshotState['nodes'][number], nodes: SnapshotState['nodes'], + platform: Platform, ): boolean { - if (node.hittable === true) return true; if (hasPositiveRect(node.rect)) return isRectVisibleInViewport(node, nodes); if (node.rect) return false; + if (platform !== 'android' && node.hittable === true) return true; const anchor = resolveVisibilityAnchor(node, nodes); if (!anchor) return false; - if (anchor.hittable === true) return true; - if (!hasPositiveRect(anchor.rect)) return false; + if (!hasPositiveRect(anchor.rect)) return platform !== 'android' && anchor.hittable === true; return isRectVisibleInViewport(anchor, nodes); } diff --git a/src/utils/snapshot-label-signals.ts b/src/utils/snapshot-label-signals.ts index 19f9d8b0c..2f4727078 100644 --- a/src/utils/snapshot-label-signals.ts +++ b/src/utils/snapshot-label-signals.ts @@ -4,6 +4,6 @@ export function normalizeRepeatedNodeLabel(label: string): string | null { return normalized; } -export function isEmailLikeLabel(label: string): boolean { +function isEmailLikeLabel(label: string): boolean { return /\S+@\S+\.\S+/.test(label); } diff --git a/test/integration/provider-scenarios/android-find.test.ts b/test/integration/provider-scenarios/android-find.test.ts index 19ca5bb91..67fb1160e 100644 --- a/test/integration/provider-scenarios/android-find.test.ts +++ b/test/integration/provider-scenarios/android-find.test.ts @@ -2,7 +2,7 @@ import assert from 'node:assert/strict'; import { test } from 'vitest'; import type { AndroidAdbProvider } from '../../../src/platforms/android/adb-executor.ts'; import { arrayEqual, assertCommandCall } from './assertions.ts'; -import { androidSettingsXml } from './android-world.ts'; +import { androidSettingsXml, androidSnapshotHelperOutput } from './android-world.ts'; import { PROVIDER_SCENARIO_ANDROID } from './fixtures.ts'; import { createProviderScenarioHarness } from './harness.ts'; @@ -160,7 +160,6 @@ test('Provider-backed integration Android find flow covers refs, wait, ambiguity assert.equal(lastAppMatch.x, 122); assert.equal(lastAppMatch.y, 217); - assertCommandCall(adbCalls, ['exec-out', 'uiautomator', 'dump', '/dev/tty']); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '88', '151']); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '122', '217']); assertCommandCall(adbCalls, ['shell', 'input', 'tap', '195', '52']); @@ -197,6 +196,15 @@ function androidFindAdbResult( exitCode: 0, }; } + if (args.join(' ').startsWith('shell am instrument ')) { + return { + stdout: androidSnapshotHelperOutput( + androidSettingsXml(searchText, { duplicateAppsRow: includeDuplicateAppsRow }), + ), + stderr: '', + exitCode: 0, + }; + } return { stdout: '', stderr: '', exitCode: 0 }; } diff --git a/test/integration/provider-scenarios/android-lifecycle.test.ts b/test/integration/provider-scenarios/android-lifecycle.test.ts index a97c8957b..b4dc93d42 100644 --- a/test/integration/provider-scenarios/android-lifecycle.test.ts +++ b/test/integration/provider-scenarios/android-lifecycle.test.ts @@ -1226,6 +1226,8 @@ function assertAndroidPushAndEventContract(world: AndroidSettingsWorld): void { 'android.intent.action.VIEW', '-d', 'demo://agent-device/event?name=screenshot_taken&payload=%7B%22source%22%3A%22provider-scenario%22%2C%22foreground%22%3Atrue%7D&platform=android', + '-p', + 'com.example.demo', ]); assertCommandCall(adbCalls, ['shell', 'cmd', 'clipboard', 'get', 'text']); assertCommandCall(adbCalls, ['shell', 'cmd', 'clipboard', 'set', 'text', 'android otp']); diff --git a/test/integration/provider-scenarios/android-test-suite.test.ts b/test/integration/provider-scenarios/android-test-suite.test.ts index 70a8ea9c5..45be5e6e0 100644 --- a/test/integration/provider-scenarios/android-test-suite.test.ts +++ b/test/integration/provider-scenarios/android-test-suite.test.ts @@ -68,3 +68,72 @@ test('Provider-backed integration Android replay test suite covers retries and f assert.equal(failFastSuite.notRun, 1, JSON.stringify(failFastSuite)); }); }); + +test('Provider-backed integration Android Maestro replay uses fresh selector snapshots and content-lane swipes', async () => { + let snapshots = 0; + await withProviderScenarioResource( + async () => + await createAndroidSettingsWorld({ + snapshotXml: () => { + snapshots += 1; + return androidMaestroReplayXml( + snapshots === 1 ? '[16,24][374,80]' : '[100,300][260,360]', + ); + }, + }), + async (world) => { + const client = world.daemon.client(); + const suiteRoot = path.join(world.tempRoot, 'suite-maestro'); + fs.mkdirSync(suiteRoot, { recursive: true }); + const flowPath = path.join(suiteRoot, 'maestro-flow.yaml'); + fs.writeFileSync( + flowPath, + [ + 'appId: com.android.settings', + '---', + '- launchApp', + '- assertVisible: Apps', + '- tapOn: Search', + '- swipe:', + ' start: 90%, 50%', + ' end: 10%, 50%', + ' duration: 300', + '', + ].join('\n'), + ); + + const suite = await client.replay.test({ + paths: [flowPath], + backend: 'maestro', + artifactsDir: path.join(suiteRoot, 'artifacts'), + timeoutMs: 30000, + ...world.selection, + }); + + assert.equal(suite.total, 1, JSON.stringify(suite)); + assert.equal(suite.passed, 1, JSON.stringify(suite)); + assert.equal(suite.failed, 0, JSON.stringify(suite)); + assert.deepEqual( + world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input tap'), + ['shell', 'input', 'tap', '180', '330'], + ); + assert.deepEqual( + world.adbCalls.find((call) => call.slice(0, 3).join(' ') === 'shell input swipe'), + ['shell', 'input', 'swipe', '351', '390', '39', '390', '300'], + ); + assert.equal(snapshots >= 2, true); + }, + ); +}); + +function androidMaestroReplayXml(searchBounds: string): string { + return [ + '', + '', + ' ', + ' ', + ` `, + ' ', + '', + ].join('\n'); +} diff --git a/test/integration/provider-scenarios/android-world.ts b/test/integration/provider-scenarios/android-world.ts index 67aa25f6b..f5ecb7b2a 100644 --- a/test/integration/provider-scenarios/android-world.ts +++ b/test/integration/provider-scenarios/android-world.ts @@ -320,6 +320,13 @@ function androidCaptureAdbResult( searchText: string, snapshotXml?: () => string, ): AndroidAdbResult | undefined { + if (key.startsWith('shell am instrument ')) { + return { + stdout: androidSnapshotHelperOutput(snapshotXml?.() ?? androidSettingsXml(searchText)), + stderr: '', + exitCode: 0, + }; + } if (key === 'exec-out uiautomator dump /dev/tty') { return { stdout: snapshotXml?.() ?? androidSettingsXml(searchText), @@ -333,6 +340,22 @@ function androidCaptureAdbResult( return undefined; } +export function androidSnapshotHelperOutput(xml: string): string { + return [ + 'INSTRUMENTATION_STATUS: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_STATUS: helperApiVersion=1', + 'INSTRUMENTATION_STATUS: outputFormat=uiautomator-xml', + 'INSTRUMENTATION_STATUS: chunkIndex=0', + 'INSTRUMENTATION_STATUS: chunkCount=1', + `INSTRUMENTATION_STATUS: payloadBase64=${Buffer.from(xml, 'utf8').toString('base64')}`, + 'INSTRUMENTATION_STATUS_CODE: 1', + 'INSTRUMENTATION_RESULT: agentDeviceProtocol=android-snapshot-helper-v1', + 'INSTRUMENTATION_RESULT: helperApiVersion=1', + 'INSTRUMENTATION_RESULT: ok=true', + 'INSTRUMENTATION_CODE: 0', + ].join('\n'); +} + export function androidSettingsXml( searchText: string, options: { duplicateAppsRow?: boolean } = {}, diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index 41d7244de..a599d244d 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -1159,7 +1159,7 @@ const SKILL_GUIDANCE_CASES: Case[] = [ 'Platform: Android', 'Launch context: Expo Go', 'Project URL: exp://10.0.2.2:8081', - 'Android does not support open ; use a URL target for deep links', + 'Android Expo URLs can be opened directly when no specific app package must be forced', ], task: 'Plan the command to launch the Expo project on Android using the project URL.', outputs: [plannedCommand('open'), /exp:\/\/10\.0\.2\.2:8081/i, /--platform android/i], diff --git a/website/docs/docs/replay-e2e.md b/website/docs/docs/replay-e2e.md index 675266000..fd44fc32c 100644 --- a/website/docs/docs/replay-e2e.md +++ b/website/docs/docs/replay-e2e.md @@ -61,11 +61,11 @@ Maestro compatibility translates supported YAML commands into Agent Device repla - Supported and unsupported capabilities: https://github.com/callstackincubator/agent-device/issues/558 - New focused compatibility request: https://github.com/callstackincubator/agent-device/issues/new -Currently supported areas include app launch without state-reset side effects, file and inline `runFlow` with `when.platform`, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn`, `doubleTapOn`, `longPressOn`, `inputText`, `pasteText`, `openLink`, visibility assertions, literal `assertTrue`, `extendedWaitUntil`, `scroll`, absolute/percentage `swipe`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, `stopApp` / `killApp`, airplane mode, mock location, orientation, supported permission targets, and screen recording. +Currently supported areas include app launch with Apple-platform launch arguments and iOS simulator `clearState`, file and inline `runFlow` with `when.platform`, `when.visible`, `when.notVisible`, and limited `when.true` boolean/platform expressions, `onFlowStart` / `onFlowComplete`, deterministic `repeat.times`, `tapOn` including `optional`, `index`, `childOf`, `label`, and absolute/percentage point taps, `doubleTapOn`, `longPressOn`, `inputText`, focused-field `eraseText`, `pasteText`, `openLink`, visibility assertions, `extendedWaitUntil`, `scroll`, `scrollUntilVisible`, absolute/percentage `swipe`, `swipe.label`, screenshots, keyboard dismiss, basic `pressKey`, `back`, animation waits, and `stopApp`. `runScript` is supported only as an ordered Maestro compatibility step for trusted file/env scripts that use `http.post`, `json`, and `output` variables; it can make network requests, and is not a native `.ad` command or security sandbox. Script execution uses Node `vm` only for compatibility isolation, not for security; the script timeout bounds synchronous execution, while `http.post` requests are bounded by the helper process timeout. Output keys cannot contain `.` because exported variables are addressed as `output.`. Maestro `env` values use the same replay precedence as `.ad` files: flow `env` is the default, shell `AD_VAR_*` values override it, and CLI `-e KEY=VALUE` wins over both. -Runtime-dependent Maestro features such as `scrollUntilVisible`, `repeat.while`, `runFlow.when.visible`, `runScript`, `evalScript`, text clearing, and app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. +Unsupported Maestro features such as `repeat.while`, full expression predicates beyond boolean literals and `maestro.platform` comparisons, `evalScript`, device utility commands, Android app launch arguments, and Android app state reset are tracked separately because they require neutral Agent Device runtime or device capabilities before they can be mapped safely. ## Run a lightweight `.ad` suite