diff --git a/apps/desktop/src/main/services/macosVm/macosVmService.ts b/apps/desktop/src/main/services/macosVm/macosVmService.ts index fe7c956e3..8f14f26f3 100644 --- a/apps/desktop/src/main/services/macosVm/macosVmService.ts +++ b/apps/desktop/src/main/services/macosVm/macosVmService.ts @@ -1446,7 +1446,10 @@ export function createMacosVmService(args: CreateMacosVmServiceArgs) { const screenshot = await directVncClient.captureScreenshot(directVnc, 15_000); fs.writeFileSync(outputPath, screenshot.pngData); const target = directVncTargetForRecord(lane, record, screenshot); - emitOperation("screenshot", "completed", lane.id, record.name, "Captured macOS VM screenshot through headless VNC."); + const retryDetail = screenshot.blankFrameAttempts && screenshot.blankFrameAttempts > 0 + ? ` after waiting through ${screenshot.blankFrameAttempts} black frame${screenshot.blankFrameAttempts === 1 ? "" : "s"}` + : ""; + emitOperation("screenshot", "completed", lane.id, record.name, `Captured macOS VM screenshot through headless VNC${retryDetail}.`); return { ok: true, laneId: lane.id, diff --git a/apps/desktop/src/main/services/macosVm/rfbDirectClient.test.ts b/apps/desktop/src/main/services/macosVm/rfbDirectClient.test.ts new file mode 100644 index 000000000..378a67099 --- /dev/null +++ b/apps/desktop/src/main/services/macosVm/rfbDirectClient.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { isLikelyBlankRgbaFrame } from "./rfbDirectClient"; + +function rgbaFrame(width: number, height: number, fill: [number, number, number, number]): Buffer { + const buffer = Buffer.alloc(width * height * 4); + for (let offset = 0; offset < buffer.length; offset += 4) { + buffer[offset] = fill[0]; + buffer[offset + 1] = fill[1]; + buffer[offset + 2] = fill[2]; + buffer[offset + 3] = fill[3]; + } + return buffer; +} + +describe("isLikelyBlankRgbaFrame", () => { + it("treats all-black and transparent VNC frames as blank", () => { + expect(isLikelyBlankRgbaFrame(64, 64, rgbaFrame(64, 64, [0, 0, 0, 255]))).toBe(true); + expect(isLikelyBlankRgbaFrame(64, 64, rgbaFrame(64, 64, [0, 0, 0, 0]))).toBe(true); + }); + + it("still treats a black frame with only a tiny cursor-sized patch as blank", () => { + const frame = rgbaFrame(320, 200, [0, 0, 0, 255]); + for (let y = 0; y < 6; y += 1) { + for (let x = 0; x < 6; x += 1) { + const offset = ((y * 320) + x) * 4; + frame[offset] = 255; + frame[offset + 1] = 255; + frame[offset + 2] = 255; + frame[offset + 3] = 255; + } + } + + expect(isLikelyBlankRgbaFrame(320, 200, frame)).toBe(true); + }); + + it("treats transparent black frames with only a noisy pixel as blank", () => { + const frame = rgbaFrame(1440, 900, [0, 0, 0, 0]); + frame[0] = 13; + frame[1] = 14; + frame[2] = 4; + frame[3] = 255; + + expect(isLikelyBlankRgbaFrame(1440, 900, frame)).toBe(true); + }); + + it("keeps dark real content once enough non-black pixels are present", () => { + const frame = rgbaFrame(320, 200, [0, 0, 0, 255]); + for (let y = 20; y < 60; y += 1) { + for (let x = 30; x < 220; x += 1) { + const offset = ((y * 320) + x) * 4; + frame[offset] = 18; + frame[offset + 1] = 21; + frame[offset + 2] = 27; + frame[offset + 3] = 255; + } + } + + expect(isLikelyBlankRgbaFrame(320, 200, frame)).toBe(false); + }); +}); diff --git a/apps/desktop/src/main/services/macosVm/rfbDirectClient.ts b/apps/desktop/src/main/services/macosVm/rfbDirectClient.ts index 38cda66dd..e7a692631 100644 --- a/apps/desktop/src/main/services/macosVm/rfbDirectClient.ts +++ b/apps/desktop/src/main/services/macosVm/rfbDirectClient.ts @@ -11,10 +11,15 @@ export type DirectVncScreenshot = { width: number; height: number; pngData: Buffer; + blankFrameAttempts?: number; }; const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); const MOUSE_LEFT_BUTTON = 1; +const KEYSYM_SHIFT_LEFT = 0xffe1; +const MAX_FRAME_ANALYSIS_SAMPLES = 4096; +const BLANK_FRAME_NON_BLACK_RATIO = 0.003; +const BLANK_FRAME_RETRY_MS = 250; function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); @@ -75,6 +80,33 @@ function encodeRgbaPng(width: number, height: number, rgba: Buffer): Buffer { ]); } +function makeOpaqueRgba(width: number, height: number, rgba: Buffer): Buffer { + const length = Math.max(0, Math.floor(width) * Math.floor(height) * 4); + const opaque = Buffer.from(rgba.subarray(0, length)); + for (let offset = 3; offset < opaque.length; offset += 4) { + opaque[offset] = 255; + } + return opaque; +} + +export function isLikelyBlankRgbaFrame(width: number, height: number, rgba: Buffer): boolean { + const pixelCount = Math.max(0, Math.floor(width) * Math.floor(height)); + if (pixelCount <= 0 || rgba.length < pixelCount * 4) return true; + const stride = Math.max(1, Math.floor(pixelCount / MAX_FRAME_ANALYSIS_SAMPLES)); + let sampled = 0; + let nonBlack = 0; + for (let pixel = 0; pixel < pixelCount; pixel += stride) { + const offset = pixel * 4; + const red = rgba[offset] ?? 0; + const green = rgba[offset + 1] ?? 0; + const blue = rgba[offset + 2] ?? 0; + sampled += 1; + if (red > 12 || green > 12 || blue > 12) nonBlack += 1; + } + if (sampled === 0) return true; + return nonBlack / sampled < BLANK_FRAME_NON_BLACK_RATIO; +} + function createClient(): VncClient { const client = new VncClient({ debug: false, @@ -100,6 +132,16 @@ function createClient(): VncClient { return client; } +function wakeVncDisplay(client: VncClient, width: number, height: number): void { + try { + client.sendPointerEvent(Math.floor(width / 2), Math.floor(height / 2), 0); + client.sendKeyEvent(KEYSYM_SHIFT_LEFT, true); + client.sendKeyEvent(KEYSYM_SHIFT_LEFT, false); + } catch { + // Display wake-up is best-effort; screenshot capture still retries frames. + } +} + function connectVnc(connection: DirectVncConnection, timeoutMs: number): Promise { return new Promise((resolve, reject) => { const client = createClient(); @@ -157,49 +199,82 @@ export async function captureVncScreenshot( ): Promise { return await withVncClient(connection, timeoutMs, async (client) => new Promise((resolve, reject) => { let settled = false; - const timeout = setTimeout(() => finish(new Error(`Timed out capturing VNC screenshot from ${connection.host}:${connection.port}.`)), timeoutMs); + let blankFrameAttempts = 0; + let retryTimer: NodeJS.Timeout | null = null; + const requestFullFrame = (): void => { + client.requestFrameUpdate(true, 0, 0, 0, Math.max(1, client.clientWidth), Math.max(1, client.clientHeight)); + }; + const timeout = setTimeout(() => { + const suffix = blankFrameAttempts > 0 + ? ` VNC returned ${blankFrameAttempts} black frame${blankFrameAttempts === 1 ? "" : "s"} while ADE waited, so ADE skipped attaching the unusable frame. The VM display may be asleep or still booting; try the screenshot again in a few seconds.` + : ""; + finishWithError(new Error(`Timed out capturing VNC screenshot from ${connection.host}:${connection.port}.${suffix}`)); + }, timeoutMs); const cleanup = (): void => { clearTimeout(timeout); + if (retryTimer) clearTimeout(retryTimer); client.removeListener("firstFrameUpdate", onFrame); client.removeListener("frameUpdated", onFrame); client.removeListener("connectError", onError); client.removeListener("authError", onAuthError); client.removeListener("closed", onClosed); }; - const finish = (error?: Error): void => { + const finishWithError = (error: Error): void => { if (settled) return; settled = true; cleanup(); - if (error) { - reject(error); - return; - } - const width = Math.max(1, client.clientWidth); - const height = Math.max(1, client.clientHeight); - resolve({ - width, - height, - pngData: encodeRgbaPng(width, height, Buffer.from(client.fb)), - }); + reject(error); + }; + const finishWithScreenshot = (screenshot: DirectVncScreenshot): void => { + if (settled) return; + settled = true; + cleanup(); + resolve(screenshot); }; function onFrame(): void { - finish(); + const width = Math.max(1, client.clientWidth); + const height = Math.max(1, client.clientHeight); + const rgba = Buffer.from(client.fb); + if (!isLikelyBlankRgbaFrame(width, height, rgba)) { + finishWithScreenshot({ + width, + height, + pngData: encodeRgbaPng(width, height, makeOpaqueRgba(width, height, rgba)), + blankFrameAttempts, + }); + return; + } + + blankFrameAttempts += 1; + if (blankFrameAttempts === 1 || blankFrameAttempts % 3 === 0) { + wakeVncDisplay(client, width, height); + } + if (!retryTimer) { + retryTimer = setTimeout(() => { + retryTimer = null; + try { + requestFullFrame(); + } catch (error) { + onError(error); + } + }, BLANK_FRAME_RETRY_MS); + } } function onError(error: unknown): void { - finish(error instanceof Error ? error : new Error(String(error))); + finishWithError(error instanceof Error ? error : new Error(String(error))); } function onAuthError(): void { - finish(new Error(`VNC authentication failed for ${connection.host}:${connection.port}.`)); + finishWithError(new Error(`VNC authentication failed for ${connection.host}:${connection.port}.`)); } function onClosed(): void { - finish(new Error(`VNC connection closed before a screenshot was captured from ${connection.host}:${connection.port}.`)); + finishWithError(new Error(`VNC connection closed before a screenshot was captured from ${connection.host}:${connection.port}.`)); } - client.once("firstFrameUpdate", onFrame); - client.once("frameUpdated", onFrame); + client.on("firstFrameUpdate", onFrame); + client.on("frameUpdated", onFrame); client.once("connectError", onError); client.once("authError", onAuthError); client.once("closed", onClosed); - client.requestFrameUpdate(true, 0, 0, 0, Math.max(1, client.clientWidth), Math.max(1, client.clientHeight)); + requestFullFrame(); })); } diff --git a/apps/desktop/src/renderer/components/terminals/useWorkSessions.test.ts b/apps/desktop/src/renderer/components/terminals/useWorkSessions.test.ts index f888c48a9..4a4742719 100644 --- a/apps/desktop/src/renderer/components/terminals/useWorkSessions.test.ts +++ b/apps/desktop/src/renderer/components/terminals/useWorkSessions.test.ts @@ -9,6 +9,7 @@ import { act, renderHook, waitFor } from "@testing-library/react"; const focusSessionSpy = vi.fn(); const selectLaneSpy = vi.fn(); const setWorkViewStateSpy = vi.fn(); +const refreshLanesSpy = vi.fn(); const navigateSpy = vi.fn(); let fakeAppStoreState: Record; const routerLocation = { @@ -24,6 +25,7 @@ function resetFakeAppStoreState() { focusSession: focusSessionSpy, focusedSessionId: null, selectLane: selectLaneSpy, + refreshLanes: refreshLanesSpy.mockResolvedValue(undefined), workViewByProject: {}, setWorkViewState: setWorkViewStateSpy, }; @@ -260,6 +262,55 @@ describe("useWorkSessions — refresh-before-focus ordering", () => { expect(refreshDoneIdx).toBeLessThan(openTabIdx); }); + it("lightly refreshes lanes when Work sessions load before lane state recovers", async () => { + fakeAppStoreState = { + ...fakeAppStoreState, + lanes: [], + }; + listSessionsCachedMock.mockResolvedValue([ + makeSession("session-1", "lane-1"), + ]); + + renderHook(() => useWorkSessions()); + + await waitFor(() => { + expect(refreshLanesSpy).toHaveBeenCalledWith({ + includeStatus: false, + includeSnapshots: false, + includeConflictStatus: false, + includeRebaseSuggestions: false, + includeAutoRebaseStatus: false, + }); + }); + expect(refreshLanesSpy).toHaveBeenCalledTimes(1); + }); + + it("does not retry lane recovery on every session refresh after a failure", async () => { + fakeAppStoreState = { + ...fakeAppStoreState, + lanes: [], + }; + refreshLanesSpy.mockRejectedValue(new Error("IPC unavailable")); + listSessionsCachedMock.mockResolvedValue([ + makeSession("session-1", "lane-1"), + ]); + + const { result } = renderHook(() => useWorkSessions()); + + await waitFor(() => { + expect(refreshLanesSpy).toHaveBeenCalledTimes(1); + }); + + listSessionsCachedMock.mockResolvedValue([ + makeSession("session-2", "lane-1"), + ]); + await act(async () => { + await result.current.refresh({ force: true }); + }); + + expect(refreshLanesSpy).toHaveBeenCalledTimes(1); + }); + it("launchPtySession keeps the new terminal visible when the forced refresh is stale", async () => { const workState = { openItemIds: [] as string[], diff --git a/apps/desktop/src/renderer/components/terminals/useWorkSessions.ts b/apps/desktop/src/renderer/components/terminals/useWorkSessions.ts index 8547a1b99..d95d5c88e 100644 --- a/apps/desktop/src/renderer/components/terminals/useWorkSessions.ts +++ b/apps/desktop/src/renderer/components/terminals/useWorkSessions.ts @@ -322,6 +322,7 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {}) const lanes = useAppStore((s) => s.lanes); const focusSession = useAppStore((s) => s.focusSession); const selectLane = useAppStore((s) => s.selectLane); + const refreshLanes = useAppStore((s) => s.refreshLanes); const workViewByProject = useAppStore((s) => s.workViewByProject); const setWorkViewState = useAppStore((s) => s.setWorkViewState); @@ -338,6 +339,7 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {}) const partiallyAppliedUrlFilterKeyRef = useRef(null); const hasLoadedOnceRef = useRef(false); const projectRootRef = useRef(projectRoot); + const laneRecoveryRefreshProjectRef = useRef(null); const isWorkRoute = active && (location.pathname === "/work" || location.pathname.startsWith("/work/")); useEffect(() => { @@ -386,6 +388,10 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {}) for (const session of sessions) map.set(session.id, session); return map; }, [sessions]); + const hasLaneBackedSessions = useMemo( + () => sessions.some((session) => Boolean(session.laneId)), + [sessions], + ); const selectLaneForActiveTab = useCallback( (sessionId: string | null) => { @@ -794,12 +800,31 @@ export function useWorkSessions({ active = true }: UseWorkSessionsOptions = {}) } hasLoadedOnceRef.current = false; hasRunningSessionsRef.current = false; + laneRecoveryRefreshProjectRef.current = null; appliedQuerySessionIdRef.current = null; appliedUrlFilterKeyRef.current = null; partiallyAppliedUrlFilterKeyRef.current = null; pendingOptimisticSessionsRef.current.clear(); }, [projectRoot]); + useEffect(() => { + if (!projectRoot || !isWorkRoute) return; + if (lanes.length > 0) { + laneRecoveryRefreshProjectRef.current = null; + return; + } + if (!hasLaneBackedSessions) return; + if (laneRecoveryRefreshProjectRef.current === projectRoot) return; + laneRecoveryRefreshProjectRef.current = projectRoot; + void refreshLanes({ + includeStatus: false, + includeSnapshots: false, + includeConflictStatus: false, + includeRebaseSuggestions: false, + includeAutoRebaseStatus: false, + }).catch(() => {}); + }, [hasLaneBackedSessions, isWorkRoute, lanes.length, projectRoot, refreshLanes]); + useEffect(() => { if (!projectRoot || !isWorkRoute) return; const isInitialLoad = !hasLoadedOnceRef.current; diff --git a/docs/features/computer-use/README.md b/docs/features/computer-use/README.md index be8a756c5..a7be12655 100644 --- a/docs/features/computer-use/README.md +++ b/docs/features/computer-use/README.md @@ -138,7 +138,10 @@ Control flows through `ade.macosVm.*` IPC and the `macos_vm` ADE action domain: `getSharePolicy`, `focusWindow`, `captureScreenshot`, `selectPoint`, `click`, and `typeText`. The Work sidebar's Mac VM tab renders the desktop panel, while `ade --socket macos-vm ...` gives agents the same status/start/screenshot/select -and click/type controls from the CLI. +and click/type controls from the CLI. Headless screenshot capture uses direct +VNC first; ADE waits through transient black framebuffer updates, wakes the +display with a pointer move plus a Shift key tap, and refuses to attach a +persistent black frame as review context. ## Cross-links diff --git a/docs/features/terminals-and-sessions/README.md b/docs/features/terminals-and-sessions/README.md index d3b3ffd60..c748ad687 100644 --- a/docs/features/terminals-and-sessions/README.md +++ b/docs/features/terminals-and-sessions/README.md @@ -344,7 +344,13 @@ Renderer surfaces: - `apps/desktop/src/main/services/macosVm/rfbDirectClient.ts` — headless VNC bridge for screenshot, click, and type operations. It disables unsupported audio negotiation for Lume VNC sessions and - encodes captured RGBA frames as PNGs for proof/context flows. + encodes captured RGBA frames as PNGs for proof/context flows. Screenshot + capture waits through transient black frames, wakes the VNC display with a + pointer move plus a Shift key tap, and fails instead of returning a persistent + black frame as usable context. +- `apps/desktop/src/main/services/macosVm/rfbDirectClient.test.ts` — + focused coverage for direct-VNC blank-frame detection so boot/sleep frames + do not become misleading screenshot proof. - `apps/desktop/src/main/services/macosVm/macosVmService.test.ts` — macOS VM provider, share-policy, lifecycle, guidance, and direct-VNC control tests.