Harden Mac VM screenshots against blank VNC frames

apps/desktop/src/main/services/macosVm/macosVmService.ts

@@ -1446,7 +1446,10 @@ export function createMacosVmService(args: CreateMacosVmServiceArgs) {
       const screenshot = await directVncClient.captureScreenshot(directVnc, 15_000);
       fs.writeFileSync(outputPath, screenshot.pngData);
       const target = directVncTargetForRecord(lane, record, screenshot);
-      emitOperation("screenshot", "completed", lane.id, record.name, "Captured macOS VM screenshot through headless VNC.");
+      const retryDetail = screenshot.blankFrameAttempts && screenshot.blankFrameAttempts > 0
+        ? ` after waiting through ${screenshot.blankFrameAttempts} black frame${screenshot.blankFrameAttempts === 1 ? "" : "s"}`
+        : "";
+      emitOperation("screenshot", "completed", lane.id, record.name, `Captured macOS VM screenshot through headless VNC${retryDetail}.`);
       return {
         ok: true,
         laneId: lane.id,

apps/desktop/src/main/services/macosVm/rfbDirectClient.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, it } from "vitest";
+import { isLikelyBlankRgbaFrame } from "./rfbDirectClient";
+
+function rgbaFrame(width: number, height: number, fill: [number, number, number, number]): Buffer {
+  const buffer = Buffer.alloc(width * height * 4);
+  for (let offset = 0; offset < buffer.length; offset += 4) {
+    buffer[offset] = fill[0];
+    buffer[offset + 1] = fill[1];
+    buffer[offset + 2] = fill[2];
+    buffer[offset + 3] = fill[3];
+  }
+  return buffer;
+}
+
+describe("isLikelyBlankRgbaFrame", () => {
+  it("treats all-black and transparent VNC frames as blank", () => {
+    expect(isLikelyBlankRgbaFrame(64, 64, rgbaFrame(64, 64, [0, 0, 0, 255]))).toBe(true);
+    expect(isLikelyBlankRgbaFrame(64, 64, rgbaFrame(64, 64, [0, 0, 0, 0]))).toBe(true);
+  });
+
+  it("still treats a black frame with only a tiny cursor-sized patch as blank", () => {
+    const frame = rgbaFrame(320, 200, [0, 0, 0, 255]);
+    for (let y = 0; y < 6; y += 1) {
+      for (let x = 0; x < 6; x += 1) {
+        const offset = ((y * 320) + x) * 4;
+        frame[offset] = 255;
+        frame[offset + 1] = 255;
+        frame[offset + 2] = 255;
+        frame[offset + 3] = 255;
+      }
+    }
+
+    expect(isLikelyBlankRgbaFrame(320, 200, frame)).toBe(true);
+  });
+
+  it("treats transparent black frames with only a noisy pixel as blank", () => {
+    const frame = rgbaFrame(1440, 900, [0, 0, 0, 0]);
+    frame[0] = 13;
+    frame[1] = 14;
+    frame[2] = 4;
+    frame[3] = 255;
+
+    expect(isLikelyBlankRgbaFrame(1440, 900, frame)).toBe(true);
+  });
+
+  it("keeps dark real content once enough non-black pixels are present", () => {
+    const frame = rgbaFrame(320, 200, [0, 0, 0, 255]);
+    for (let y = 20; y < 60; y += 1) {
+      for (let x = 30; x < 220; x += 1) {
+        const offset = ((y * 320) + x) * 4;
+        frame[offset] = 18;
+        frame[offset + 1] = 21;
+        frame[offset + 2] = 27;
+        frame[offset + 3] = 255;
+      }
+    }
+
+    expect(isLikelyBlankRgbaFrame(320, 200, frame)).toBe(false);
+  });
+});

apps/desktop/src/main/services/macosVm/rfbDirectClient.ts

@@ -11,10 +11,15 @@ export type DirectVncScreenshot = {
   width: number;
   height: number;
   pngData: Buffer;
+  blankFrameAttempts?: number;
 };
 
 const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
 const MOUSE_LEFT_BUTTON = 1;
+const KEYSYM_SHIFT_LEFT = 0xffe1;
+const MAX_FRAME_ANALYSIS_SAMPLES = 4096;
+const BLANK_FRAME_NON_BLACK_RATIO = 0.003;
+const BLANK_FRAME_RETRY_MS = 250;
 
 function delay(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
@@ -75,6 +80,33 @@ function encodeRgbaPng(width: number, height: number, rgba: Buffer): Buffer {
   ]);
 }
 
+function makeOpaqueRgba(width: number, height: number, rgba: Buffer): Buffer {
+  const length = Math.max(0, Math.floor(width) * Math.floor(height) * 4);
+  const opaque = Buffer.from(rgba.subarray(0, length));
+  for (let offset = 3; offset < opaque.length; offset += 4) {
+    opaque[offset] = 255;
+  }
+  return opaque;
+}
+
+export function isLikelyBlankRgbaFrame(width: number, height: number, rgba: Buffer): boolean {
+  const pixelCount = Math.max(0, Math.floor(width) * Math.floor(height));
+  if (pixelCount <= 0 || rgba.length < pixelCount * 4) return true;
+  const stride = Math.max(1, Math.floor(pixelCount / MAX_FRAME_ANALYSIS_SAMPLES));
+  let sampled = 0;
+  let nonBlack = 0;
+  for (let pixel = 0; pixel < pixelCount; pixel += stride) {
+    const offset = pixel * 4;
+    const red = rgba[offset] ?? 0;
+    const green = rgba[offset + 1] ?? 0;
+    const blue = rgba[offset + 2] ?? 0;
+    sampled += 1;
+    if (red > 12 || green > 12 || blue > 12) nonBlack += 1;
+  }
+  if (sampled === 0) return true;
+  return nonBlack / sampled < BLANK_FRAME_NON_BLACK_RATIO;
+}
+
 function createClient(): VncClient {
   const client = new VncClient({
     debug: false,
@@ -100,6 +132,16 @@ function createClient(): VncClient {
   return client;
 }
 
+function wakeVncDisplay(client: VncClient, width: number, height: number): void {
+  try {
+    client.sendPointerEvent(Math.floor(width / 2), Math.floor(height / 2), 0);
+    client.sendKeyEvent(KEYSYM_SHIFT_LEFT, true);
+    client.sendKeyEvent(KEYSYM_SHIFT_LEFT, false);
+  } catch {
+    // Display wake-up is best-effort; screenshot capture still retries frames.
+  }
+}
+
 function connectVnc(connection: DirectVncConnection, timeoutMs: number): Promise<VncClient> {
   return new Promise((resolve, reject) => {
     const client = createClient();
@@ -157,49 +199,82 @@ export async function captureVncScreenshot(
 ): Promise<DirectVncScreenshot> {
   return await withVncClient(connection, timeoutMs, async (client) => new Promise<DirectVncScreenshot>((resolve, reject) => {
     let settled = false;
-    const timeout = setTimeout(() => finish(new Error(`Timed out capturing VNC screenshot from ${connection.host}:${connection.port}.`)), timeoutMs);
+    let blankFrameAttempts = 0;
+    let retryTimer: NodeJS.Timeout | null = null;
+    const requestFullFrame = (): void => {
+      client.requestFrameUpdate(true, 0, 0, 0, Math.max(1, client.clientWidth), Math.max(1, client.clientHeight));
+    };
+    const timeout = setTimeout(() => {
+      const suffix = blankFrameAttempts > 0
+        ? ` VNC returned ${blankFrameAttempts} black frame${blankFrameAttempts === 1 ? "" : "s"} while ADE waited, so ADE skipped attaching the unusable frame. The VM display may be asleep or still booting; try the screenshot again in a few seconds.`
+        : "";
+      finishWithError(new Error(`Timed out capturing VNC screenshot from ${connection.host}:${connection.port}.${suffix}`));
+    }, timeoutMs);
     const cleanup = (): void => {
       clearTimeout(timeout);
+      if (retryTimer) clearTimeout(retryTimer);
       client.removeListener("firstFrameUpdate", onFrame);
       client.removeListener("frameUpdated", onFrame);
       client.removeListener("connectError", onError);
       client.removeListener("authError", onAuthError);
       client.removeListener("closed", onClosed);
     };
-    const finish = (error?: Error): void => {
+    const finishWithError = (error: Error): void => {
       if (settled) return;
       settled = true;
       cleanup();
-      if (error) {
-        reject(error);
-        return;
-      }
-      const width = Math.max(1, client.clientWidth);
-      const height = Math.max(1, client.clientHeight);
-      resolve({
-        width,
-        height,
-        pngData: encodeRgbaPng(width, height, Buffer.from(client.fb)),
-      });
+      reject(error);
+    };
+    const finishWithScreenshot = (screenshot: DirectVncScreenshot): void => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      resolve(screenshot);
     };
     function onFrame(): void {
-      finish();
+      const width = Math.max(1, client.clientWidth);
+      const height = Math.max(1, client.clientHeight);
+      const rgba = Buffer.from(client.fb);
+      if (!isLikelyBlankRgbaFrame(width, height, rgba)) {
+        finishWithScreenshot({
+          width,
+          height,
+          pngData: encodeRgbaPng(width, height, makeOpaqueRgba(width, height, rgba)),
+          blankFrameAttempts,
+        });
+        return;
+      }
+
+      blankFrameAttempts += 1;
+      if (blankFrameAttempts === 1 || blankFrameAttempts % 3 === 0) {
+        wakeVncDisplay(client, width, height);
+      }
+      if (!retryTimer) {
+        retryTimer = setTimeout(() => {
+          retryTimer = null;
+          try {
+            requestFullFrame();
+          } catch (error) {
+            onError(error);
+          }
+        }, BLANK_FRAME_RETRY_MS);
+      }
     }
     function onError(error: unknown): void {
-      finish(error instanceof Error ? error : new Error(String(error)));
+      finishWithError(error instanceof Error ? error : new Error(String(error)));
     }
     function onAuthError(): void {
-      finish(new Error(`VNC authentication failed for ${connection.host}:${connection.port}.`));
+      finishWithError(new Error(`VNC authentication failed for ${connection.host}:${connection.port}.`));
     }
     function onClosed(): void {
-      finish(new Error(`VNC connection closed before a screenshot was captured from ${connection.host}:${connection.port}.`));
+      finishWithError(new Error(`VNC connection closed before a screenshot was captured from ${connection.host}:${connection.port}.`));
     }
-    client.once("firstFrameUpdate", onFrame);
-    client.once("frameUpdated", onFrame);
+    client.on("firstFrameUpdate", onFrame);
+    client.on("frameUpdated", onFrame);
     client.once("connectError", onError);
     client.once("authError", onAuthError);
     client.once("closed", onClosed);
-    client.requestFrameUpdate(true, 0, 0, 0, Math.max(1, client.clientWidth), Math.max(1, client.clientHeight));
+    requestFullFrame();
   }));
 }
 

apps/desktop/src/renderer/components/terminals/useWorkSessions.test.ts

@@ -9,6 +9,7 @@ import { act, renderHook, waitFor } from "@testing-library/react";
 const focusSessionSpy = vi.fn();
 const selectLaneSpy = vi.fn();
 const setWorkViewStateSpy = vi.fn();
+const refreshLanesSpy = vi.fn();
 const navigateSpy = vi.fn();
 let fakeAppStoreState: Record<string, unknown>;
 const routerLocation = {
@@ -24,6 +25,7 @@ function resetFakeAppStoreState() {
     focusSession: focusSessionSpy,
     focusedSessionId: null,
     selectLane: selectLaneSpy,
+    refreshLanes: refreshLanesSpy.mockResolvedValue(undefined),
     workViewByProject: {},
     setWorkViewState: setWorkViewStateSpy,
   };
@@ -260,6 +262,55 @@ describe("useWorkSessions — refresh-before-focus ordering", () => {
     expect(refreshDoneIdx).toBeLessThan(openTabIdx);
   });
 
+  it("lightly refreshes lanes when Work sessions load before lane state recovers", async () => {
+    fakeAppStoreState = {
+      ...fakeAppStoreState,
+      lanes: [],
+    };
+    listSessionsCachedMock.mockResolvedValue([
+      makeSession("session-1", "lane-1"),
+    ]);
+
+    renderHook(() => useWorkSessions());
+
+    await waitFor(() => {
+      expect(refreshLanesSpy).toHaveBeenCalledWith({
+        includeStatus: false,
+        includeSnapshots: false,
+        includeConflictStatus: false,
+        includeRebaseSuggestions: false,
+        includeAutoRebaseStatus: false,
+      });
+    });
+    expect(refreshLanesSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry lane recovery on every session refresh after a failure", async () => {
+    fakeAppStoreState = {
+      ...fakeAppStoreState,
+      lanes: [],
+    };
+    refreshLanesSpy.mockRejectedValue(new Error("IPC unavailable"));
+    listSessionsCachedMock.mockResolvedValue([
+      makeSession("session-1", "lane-1"),
+    ]);
+
+    const { result } = renderHook(() => useWorkSessions());
+
+    await waitFor(() => {
+      expect(refreshLanesSpy).toHaveBeenCalledTimes(1);
+    });
+
+    listSessionsCachedMock.mockResolvedValue([
+      makeSession("session-2", "lane-1"),
+    ]);
+    await act(async () => {
+      await result.current.refresh({ force: true });
+    });
+
+    expect(refreshLanesSpy).toHaveBeenCalledTimes(1);
+  });
+
   it("launchPtySession keeps the new terminal visible when the forced refresh is stale", async () => {
     const workState = {
       openItemIds: [] as string[],

apps/desktop/src/renderer/components/terminals/useWorkSessions.ts

@@ -322,6 +322,7 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {})
   const lanes = useAppStore((s) => s.lanes);
   const focusSession = useAppStore((s) => s.focusSession);
   const selectLane = useAppStore((s) => s.selectLane);
+  const refreshLanes = useAppStore((s) => s.refreshLanes);
   const workViewByProject = useAppStore((s) => s.workViewByProject);
   const setWorkViewState = useAppStore((s) => s.setWorkViewState);
 
@@ -338,6 +339,7 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {})
   const partiallyAppliedUrlFilterKeyRef = useRef<string | null>(null);
   const hasLoadedOnceRef = useRef(false);
   const projectRootRef = useRef<string | null>(projectRoot);
+  const laneRecoveryRefreshProjectRef = useRef<string | null>(null);
   const isWorkRoute = active && (location.pathname === "/work" || location.pathname.startsWith("/work/"));
 
   useEffect(() => {
@@ -386,6 +388,10 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {})
     for (const session of sessions) map.set(session.id, session);
     return map;
   }, [sessions]);
+  const hasLaneBackedSessions = useMemo(
+    () => sessions.some((session) => Boolean(session.laneId)),
+    [sessions],
+  );
 
   const selectLaneForActiveTab = useCallback(
     (sessionId: string | null) => {
@@ -794,12 +800,31 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {})
     }
     hasLoadedOnceRef.current = false;
     hasRunningSessionsRef.current = false;
+    laneRecoveryRefreshProjectRef.current = null;
     appliedQuerySessionIdRef.current = null;
     appliedUrlFilterKeyRef.current = null;
     partiallyAppliedUrlFilterKeyRef.current = null;
     pendingOptimisticSessionsRef.current.clear();
   }, [projectRoot]);
 
+  useEffect(() => {
+    if (!projectRoot || !isWorkRoute) return;
+    if (lanes.length > 0) {
+      laneRecoveryRefreshProjectRef.current = null;
+      return;
+    }
+    if (!hasLaneBackedSessions) return;
+    if (laneRecoveryRefreshProjectRef.current === projectRoot) return;
+    laneRecoveryRefreshProjectRef.current = projectRoot;
+    void refreshLanes({
+      includeStatus: false,
+      includeSnapshots: false,
+      includeConflictStatus: false,
+      includeRebaseSuggestions: false,
+      includeAutoRebaseStatus: false,
+    }).catch(() => {});
+  }, [hasLaneBackedSessions, isWorkRoute, lanes.length, projectRoot, refreshLanes]);
+
   useEffect(() => {
     if (!projectRoot || !isWorkRoute) return;
     const isInitialLoad = !hasLoadedOnceRef.current;

docs/features/computer-use/README.md

@@ -138,7 +138,10 @@ Control flows through `ade.macosVm.*` IPC and the `macos_vm` ADE action domain:
 `getSharePolicy`, `focusWindow`, `captureScreenshot`, `selectPoint`, `click`,
 and `typeText`. The Work sidebar's Mac VM tab renders the desktop panel, while
 `ade --socket macos-vm ...` gives agents the same status/start/screenshot/select
-and click/type controls from the CLI.
+and click/type controls from the CLI. Headless screenshot capture uses direct
+VNC first; ADE waits through transient black framebuffer updates, wakes the
+display with a pointer move plus a Shift key tap, and refuses to attach a
+persistent black frame as review context.
 
 ## Cross-links