From 710a4ca013fb9253dab482c2afeeb01dd03c4657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 2 Jun 2026 11:35:59 -0700 Subject: [PATCH 1/2] fix: filter covered android snapshot surfaces --- android-snapshot-helper/README.md | 5 + .../SnapshotInstrumentation.java | 1 + src/platforms/android/__tests__/index.test.ts | 123 +++++++++++++++++- src/platforms/android/ui-hierarchy.ts | 121 ++++++++++++++++- 4 files changed, 245 insertions(+), 5 deletions(-) diff --git a/android-snapshot-helper/README.md b/android-snapshot-helper/README.md index 2808a6800..1dbb3d8b7 100644 --- a/android-snapshot-helper/README.md +++ b/android-snapshot-helper/README.md @@ -48,6 +48,11 @@ can run. The APK emits instrumentation status records using `agentDeviceProtocol=android-snapshot-helper-v1`. +The XML node attributes intentionally mirror fields consumed by the host parser, including +`visible-to-user`, `drawing-order`, bounds, text/description/id, interaction booleans, and window +metadata on window roots. `drawing-order` lets the host suppress covered same-window surfaces that +the helper traversal can receive even when they are not user-reachable. + Each XML chunk is sent with: - `outputFormat=uiautomator-xml` diff --git a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 6b4bac4bd..949021dec 100644 --- a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -513,6 +513,7 @@ private static void appendNode( appendNonEmptyAttribute(xml, "package", node.getPackageName()); appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription()); appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); + appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); appendTrueAttribute(xml, "clickable", node.isClickable()); appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled())); appendTrueAttribute(xml, "focusable", node.isFocusable()); diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index fcc619d4f..d7a7ee3c9 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -161,9 +161,17 @@ test('parseUiHierarchy decodes XML entities in Android node attributes', () => { assert.equal(result.nodes[0]!.label, 'Line 1\nLine 2\t&<>"\''); }); +test('parseUiHierarchy reads Android bounds with negative coordinates', () => { + const xml = + ''; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.deepEqual(result.nodes[0]!.rect, { x: 0, y: 935, width: 0, height: 59 }); +}); + test('androidUiNodes exposes decoded Android hierarchy metadata', () => { const xml = - ''; + ''; assert.deepEqual(Array.from(androidUiNodes(xml)), [ { @@ -177,6 +185,7 @@ test('androidUiNodes exposes decoded Android hierarchy metadata', () => { clickable: false, enabled: true, visibleToUser: true, + drawingOrder: 4, focusable: true, focused: true, password: true, @@ -272,7 +281,7 @@ test('parseUiHierarchy excludes Android nodes that are not visible to the user', ); }); -test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapshots', () => { +test('parseUiHierarchy prunes Android nodes that are not visible to the user in raw snapshots', () => { const xml = ` @@ -281,8 +290,114 @@ test('parseUiHierarchy preserves Android visible-to-user metadata in raw snapsho const result = parseUiHierarchy(xml, 800, { raw: true }); assert.equal(result.nodes[0]!.visibleToUser, true); - assert.equal(result.nodes[1]!.label, 'Hidden drawer action'); - assert.equal(result.nodes[1]!.visibleToUser, false); + assert.equal( + result.nodes.some((node) => node.label === 'Hidden drawer action'), + false, + ); +}); + +test('parseUiHierarchy prunes descendants of Android nodes that are not visible to the user', () => { + const xml = ` + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Hidden drawer action'), + false, + ); +}); + +test('parseUiHierarchy prunes lower drawing-order subtrees covered by a foreground sibling', () => { + const xml = ` + + + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Foreground action'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Hidden drawer action'), + false, + ); +}); + +test('parseUiHierarchy keeps visible side-by-side drawer and content subtrees', () => { + const xml = ` + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Visible drawer action'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Visible content action'), + true, + ); +}); + +test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unavailable', () => { + const xml = ` + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Foreground action'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Legacy drawer action'), + true, + ); +}); + +test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible overlays', () => { + const xml = ` + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Still visible action'), + true, + ); }); test('parseUiHierarchy ignores attribute-name prefix spoofing', () => { diff --git a/src/platforms/android/ui-hierarchy.ts b/src/platforms/android/ui-hierarchy.ts index c7eb18c08..87b26a509 100644 --- a/src/platforms/android/ui-hierarchy.ts +++ b/src/platforms/android/ui-hierarchy.ts @@ -17,6 +17,7 @@ export type AndroidUiNodeMetadata = { clickable?: boolean; enabled?: boolean; visibleToUser?: boolean; + drawingOrder?: number; focusable?: boolean; focused?: boolean; password?: boolean; @@ -244,6 +245,7 @@ function readNodeAttributes(node: string): Omit { focused: boolAttr('focused'), password: boolAttr('password'), ...optionalBoolAttr('visibleToUser', 'visible-to-user'), + ...optionalNumberAttr('drawingOrder', 'drawing-order'), ...optionalBoolAttr('scrollable', 'scrollable'), ...optionalBoolAttr('canScrollForward', 'can-scroll-forward'), ...optionalBoolAttr('canScrollBackward', 'can-scroll-backward'), @@ -383,7 +385,7 @@ function readXmlAttr(attrs: Map, name: string): string | null { function parseBounds(bounds: string | null): Rect | undefined { if (!bounds) return undefined; - const match = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(bounds); + const match = /\[(-?\d+),(-?\d+)\]\[(-?\d+),(-?\d+)\]/.exec(bounds); if (!match) return undefined; const x1 = Number(match[1]); const y1 = Number(match[2]); @@ -401,6 +403,7 @@ export type AndroidUiHierarchy = { rect?: Rect; enabled?: boolean; visibleToUser?: boolean; + drawingOrder?: number; hittable?: boolean; depth: number; parentIndex?: number; @@ -428,6 +431,15 @@ type AndroidNodeInclusionInfo = { isVisual: boolean; }; +type AndroidTreePruneState = { + agentVisibleContentMemo: WeakMap; +}; + +type AndroidCoveringCandidate = AndroidNode & { + rect: Rect; + drawingOrder: number; +}; + const ANDROID_WINDOW_TYPE_APPLICATION = 1; export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { @@ -461,6 +473,7 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { rect: attrs.rect, enabled: attrs.enabled, visibleToUser: attrs.visibleToUser, + drawingOrder: attrs.drawingOrder, hittable: attrs.clickable ?? attrs.focusable, scrollable: attrs.scrollable, canScrollForward: attrs.canScrollForward, @@ -481,11 +494,117 @@ export function parseUiHierarchyTree(xml: string): AndroidUiHierarchy { } match = tokenRegex.exec(xml); } + // Raw Android snapshots are uncollapsed, but still agent-visible. The helper can expose + // aria-hidden/no-hide-descendants children, so prune nodes Android marks hidden to users. + pruneAndroidInvisibleSubtrees(root); discardInactiveAndroidApplicationWindows(root); + // UiAutomation can expose covered React Native navigation surfaces in the same accessibility + // window. If a higher drawing-order sibling covers them, agents should see the foreground surface. + pruneAndroidCoveredSubtrees(root, { agentVisibleContentMemo: new WeakMap() }); applyAndroidScrollActionHints(root); return root; } +function pruneAndroidInvisibleSubtrees(node: AndroidNode): void { + let keptCount = 0; + for (const child of node.children) { + if (child.visibleToUser === false) continue; + pruneAndroidInvisibleSubtrees(child); + node.children[keptCount] = child; + keptCount += 1; + } + if (keptCount < node.children.length) { + node.children.length = keptCount; + } +} + +function pruneAndroidCoveredSubtrees(node: AndroidNode, state: AndroidTreePruneState): void { + for (const child of node.children) { + pruneAndroidCoveredSubtrees(child, state); + } + if (node.children.length < 2) { + return; + } + const siblings = node.children; + const coveringCandidates = siblings.filter((sibling) => canCoverSibling(sibling, state)); + if (coveringCandidates.length === 0) return; + node.children = siblings.filter( + (child) => !isCoveredByHigherDrawingOrderSibling(child, coveringCandidates), + ); +} + +function isCoveredByHigherDrawingOrderSibling( + node: AndroidNode, + coveringCandidates: AndroidCoveringCandidate[], +): boolean { + if (node.visibleToUser === false || node.drawingOrder === undefined || !hasPositiveRect(node)) { + return false; + } + + for (const sibling of coveringCandidates) { + if (sibling === node || sibling.drawingOrder <= node.drawingOrder) { + continue; + } + if (rectCoverage(sibling.rect, node.rect) >= 0.9) { + return true; + } + } + return false; +} + +function canCoverSibling( + node: AndroidNode, + state: AndroidTreePruneState, +): node is AndroidCoveringCandidate { + return ( + node.visibleToUser !== false && + node.drawingOrder !== undefined && + hasPositiveRect(node) && + hasAgentVisibleContent(node, state) + ); +} + +function hasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean { + const cached = state.agentVisibleContentMemo.get(node); + if (cached !== undefined) return cached; + + const result = computeHasAgentVisibleContent(node, state); + state.agentVisibleContentMemo.set(node, result); + return result; +} + +function computeHasAgentVisibleContent(node: AndroidNode, state: AndroidTreePruneState): boolean { + if (node.visibleToUser === false) return false; + if (node.hittable) return true; + const label = node.label?.trim() ?? ''; + if (label && !isGenericAndroidId(label)) return true; + const identifier = node.identifier?.trim() ?? ''; + if (identifier && !isGenericAndroidId(identifier)) return true; + return node.children.some((child) => hasAgentVisibleContent(child, state)); +} + +function hasPositiveRect(node: AndroidNode): node is AndroidNode & { rect: Rect } { + return Boolean(node.rect && node.rect.width > 0 && node.rect.height > 0); +} + +function rectCoverage(coveringRect: Rect, targetRect: Rect): number { + const targetArea = targetRect.width * targetRect.height; + if (targetArea <= 0) return 0; + return intersectionArea(coveringRect, targetRect) / targetArea; +} + +function intersectionArea(left: Rect, right: Rect): number { + const xOverlap = Math.max( + 0, + Math.min(left.x + left.width, right.x + right.width) - Math.max(left.x, right.x), + ); + const yOverlap = Math.max( + 0, + Math.min(left.y + left.height, right.y + right.height) - Math.max(left.y, right.y), + ); + return xOverlap * yOverlap; +} + function applyAndroidScrollActionHints(root: AndroidUiHierarchy): void { const stack = [...root.children]; while (stack.length > 0) { From ecf220069e326484bfb0eb80e3f6faad93af2082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 2 Jun 2026 11:46:06 -0700 Subject: [PATCH 2/2] fix: guard android drawing order metadata --- android-snapshot-helper/README.md | 3 +- .../SnapshotInstrumentation.java | 9 +++- src/platforms/android/__tests__/index.test.ts | 46 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/android-snapshot-helper/README.md b/android-snapshot-helper/README.md index 1dbb3d8b7..b30404b02 100644 --- a/android-snapshot-helper/README.md +++ b/android-snapshot-helper/README.md @@ -51,7 +51,8 @@ The APK emits instrumentation status records using The XML node attributes intentionally mirror fields consumed by the host parser, including `visible-to-user`, `drawing-order`, bounds, text/description/id, interaction booleans, and window metadata on window roots. `drawing-order` lets the host suppress covered same-window surfaces that -the helper traversal can receive even when they are not user-reachable. +the helper traversal can receive even when they are not user-reachable. The helper emits +`drawing-order` on Android API 24+ and omits it on API 23, where the platform API is unavailable. Each XML chunk is sent with: diff --git a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java index 949021dec..7f2863b7e 100644 --- a/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java +++ b/android-snapshot-helper/src/main/java/com/callstack/agentdevice/snapshothelper/SnapshotInstrumentation.java @@ -4,6 +4,7 @@ import android.app.Instrumentation; import android.app.UiAutomation; import android.graphics.Rect; +import android.os.Build; import android.os.Bundle; import android.util.Base64; import android.view.accessibility.AccessibilityNodeInfo; @@ -513,7 +514,7 @@ private static void appendNode( appendNonEmptyAttribute(xml, "package", node.getPackageName()); appendNonEmptyAttribute(xml, "content-desc", node.getContentDescription()); appendAttribute(xml, "visible-to-user", Boolean.toString(node.isVisibleToUser())); - appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); + appendDrawingOrderAttribute(xml, node); appendTrueAttribute(xml, "clickable", node.isClickable()); appendAttribute(xml, "enabled", Boolean.toString(node.isEnabled())); appendTrueAttribute(xml, "focusable", node.isFocusable()); @@ -585,6 +586,12 @@ private static void appendTrueAttribute(StringBuilder xml, String name, boolean } } + private static void appendDrawingOrderAttribute(StringBuilder xml, AccessibilityNodeInfo node) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + appendAttribute(xml, "drawing-order", Integer.toString(node.getDrawingOrder())); + } + } + private static void appendWindowMetadata(StringBuilder xml, WindowMetadata metadata) { appendAttribute(xml, "window-index", Integer.toString(metadata.index)); appendAttribute(xml, "window-type", Integer.toString(metadata.type)); diff --git a/src/platforms/android/__tests__/index.test.ts b/src/platforms/android/__tests__/index.test.ts index d7a7ee3c9..62d20e812 100644 --- a/src/platforms/android/__tests__/index.test.ts +++ b/src/platforms/android/__tests__/index.test.ts @@ -383,6 +383,52 @@ test('parseUiHierarchy keeps lower siblings when drawing-order metadata is unava ); }); +test('parseUiHierarchy keeps overlapping siblings when drawing-order ties', () => { + const xml = ` + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'First tied action'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Second tied action'), + true, + ); +}); + +test('parseUiHierarchy keeps lower siblings below the covered-area threshold', () => { + const xml = ` + + + + + + + + +`; + + const result = parseUiHierarchy(xml, 800, { raw: true }); + assert.equal( + result.nodes.some((node) => node.label === 'Partial overlay action'), + true, + ); + assert.equal( + result.nodes.some((node) => node.label === 'Mostly visible action'), + true, + ); +}); + test('parseUiHierarchy keeps lower siblings covered only by non-agent-visible overlays', () => { const xml = `