From 4400b287bd1783197f78af7cda8a800f0de82f34 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 23 Jul 2026 23:24:29 +0200 Subject: [PATCH 01/26] Add background preview capture and picture-in-picture support - Capture hidden preview frames for recording and PiP - Expose desktop IPC and automation state for PiP windows - Add focused coverage for concurrent capture and lifecycle cleanup Co-authored-by: codex --- .../src/electron/ElectronWindow.test.ts | 1 + apps/desktop/src/electron/ElectronWindow.ts | 2 + apps/desktop/src/ipc/channels.ts | 3 + apps/desktop/src/ipc/methods/preview.ts | 12 + apps/desktop/src/preload.ts | 6 + apps/desktop/src/preview-pip-preload.ts | 17 + apps/desktop/src/preview/Manager.test.ts | 238 ++++++++- apps/desktop/src/preview/Manager.ts | 487 +++++++++++++++--- apps/desktop/src/window/DesktopWindow.test.ts | 2 + apps/desktop/src/window/DesktopWindow.ts | 1 + apps/desktop/vite.config.ts | 7 + .../src/mcp/PreviewAutomationBroker.test.ts | 4 +- .../server/src/mcp/PreviewAutomationBroker.ts | 3 +- apps/server/src/mcp/toolkits/preview/tools.ts | 3 +- apps/web/src/browser/HostedBrowserWebview.tsx | 7 - apps/web/src/browser/browserRecording.test.ts | 49 +- apps/web/src/browser/browserRecording.ts | 76 ++- .../src/browser/browserRecordingScope.test.ts | 16 - apps/web/src/browser/browserRecordingScope.ts | 7 - .../preview/PreviewAutomationHosts.tsx | 14 +- .../components/preview/PreviewChromeRow.tsx | 31 ++ .../components/preview/PreviewView.test.tsx | 52 +- .../src/components/preview/PreviewView.tsx | 37 +- .../components/preview/usePreviewBridge.ts | 2 + apps/web/src/previewStateStore.test.ts | 6 + apps/web/src/previewStateStore.ts | 2 + packages/contracts/src/ipc.ts | 7 + 27 files changed, 894 insertions(+), 198 deletions(-) create mode 100644 apps/desktop/src/preview-pip-preload.ts delete mode 100644 apps/web/src/browser/browserRecordingScope.test.ts delete mode 100644 apps/web/src/browser/browserRecordingScope.ts diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index b59f8572739..67819def623 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -94,6 +94,7 @@ describe("ElectronWindow", () => { webPreferences: { preload: "/tmp/preload.js", partition: "persist:t3code-preview-test", + backgroundThrottling: null, sandbox: true, contextIsolation: true, nodeIntegration: false, diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index dacb2eebb47..4671328587a 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -22,6 +22,7 @@ const ElectronWindowCreateOptions = Schema.Struct({ webPreferences: Schema.Struct({ preload: Schema.NullOr(Schema.String), partition: Schema.NullOr(Schema.String), + backgroundThrottling: Schema.NullOr(Schema.Boolean), sandbox: Schema.NullOr(Schema.Boolean), contextIsolation: Schema.NullOr(Schema.Boolean), nodeIntegration: Schema.NullOr(Schema.Boolean), @@ -179,6 +180,7 @@ export const make = Effect.gen(function* () { webPreferences: { preload: webPreferences?.preload ?? null, partition: webPreferences?.partition ?? null, + backgroundThrottling: webPreferences?.backgroundThrottling ?? null, sandbox: webPreferences?.sandbox ?? null, contextIsolation: webPreferences?.contextIsolation ?? null, nodeIntegration: webPreferences?.nodeIntegration ?? null, diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 0d5c79cf55a..1af363d4c27 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -60,6 +60,9 @@ export const PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL = "desktop:preview-cancel-pick- export const PREVIEW_CAPTURE_SCREENSHOT_CHANNEL = "desktop:preview-capture-screenshot"; export const PREVIEW_REVEAL_ARTIFACT_CHANNEL = "desktop:preview-reveal-artifact"; export const PREVIEW_COPY_ARTIFACT_CHANNEL = "desktop:preview-copy-artifact"; +export const PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL = "desktop:preview-pip-open"; +export const PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL = "desktop:preview-pip-close"; +export const PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL = "desktop:preview-pip-frame"; export const PREVIEW_AUTOMATION_STATUS_CHANNEL = "desktop:preview-automation-status"; export const PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL = "desktop:preview-automation-snapshot"; export const PREVIEW_AUTOMATION_CLICK_CHANNEL = "desktop:preview-automation-click"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 2abf53ac284..e7a14442aae 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -158,6 +158,16 @@ export const stopRecording = tabMethod( "desktop.ipc.preview.stopRecording", (manager, tabId) => manager.stopRecording(tabId), ); +export const openPictureInPicture = tabMethod( + IpcChannels.PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL, + "desktop.ipc.preview.openPictureInPicture", + (manager, tabId) => manager.openPictureInPicture(tabId), +); +export const closePictureInPicture = tabMethod( + IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, + "desktop.ipc.preview.closePictureInPicture", + (manager, tabId) => manager.closePictureInPicture(tabId), +); export const clearCookies = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_CLEAR_COOKIES_CHANNEL, @@ -356,6 +366,8 @@ export const methods = [ captureScreenshot, revealArtifact, copyArtifactToClipboard, + openPictureInPicture, + closePictureInPicture, automationStatus, automationSnapshot, automationClick, diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 9f92cfcf2fa..89bf78ca5b1 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -177,6 +177,12 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_REVEAL_ARTIFACT_CHANNEL, { path }), copyArtifactToClipboard: (path) => ipcRenderer.invoke(IpcChannels.PREVIEW_COPY_ARTIFACT_CHANNEL, { path }), + pictureInPicture: { + open: (tabId) => + ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL, { tabId }), + close: (tabId) => + ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, { tabId }), + }, recording: { startScreencast: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId }), diff --git a/apps/desktop/src/preview-pip-preload.ts b/apps/desktop/src/preview-pip-preload.ts new file mode 100644 index 00000000000..384c4129774 --- /dev/null +++ b/apps/desktop/src/preview-pip-preload.ts @@ -0,0 +1,17 @@ +// @effect-diagnostics globalDate:off - This isolated Electron preload does not run inside an Effect runtime. +import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; +import { contextBridge, ipcRenderer } from "electron"; + +import { PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "./ipc/channels.ts"; + +contextBridge.exposeInMainWorld("previewPictureInPicture", { + onFrame: (listener: (frame: DesktopPreviewRecordingFrame) => void) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, frame: unknown) => { + if (typeof frame !== "object" || frame === null) return; + listener(frame as DesktopPreviewRecordingFrame); + }; + ipcRenderer.on(PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, wrappedListener); + }, +}); diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 6e2df118c34..375ac26460a 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1,4 +1,5 @@ import { it as effectIt } from "@effect/vitest"; +import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; import * as Effect from "effect/Effect"; @@ -20,6 +21,7 @@ import * as BrowserSession from "./BrowserSession.ts"; import * as PreviewManager from "./Manager.ts"; const { + browserWindowConstructor, createFromPath, fromId, getFocusedWebContents, @@ -29,8 +31,9 @@ const { writeFile, writeImage, } = vi.hoisted(() => ({ + browserWindowConstructor: vi.fn(), createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })), - fromId: vi.fn(() => null), + fromId: vi.fn((_id?: number) => null), getFocusedWebContents: vi.fn(() => null), mkdir: vi.fn((_path: string) => undefined), showItemInFolder: vi.fn(), @@ -40,6 +43,7 @@ const { })); vi.mock("electron", () => ({ + BrowserWindow: browserWindowConstructor, clipboard: { writeImage, }, @@ -73,6 +77,10 @@ const environmentLayer = Layer.succeed( DesktopEnvironment.DesktopEnvironment, DesktopEnvironment.DesktopEnvironment.of({ browserArtifactsDir: "/tmp/t3/dev/browser-artifacts", + dirname: "/tmp/t3/desktop", + path: { + join: (...parts: ReadonlyArray) => parts.join("/"), + }, } as DesktopEnvironment.DesktopEnvironment["Service"]), ); @@ -108,6 +116,7 @@ const withManager = ( describe("PreviewManager", () => { beforeEach(() => { + browserWindowConstructor.mockReset(); fromId.mockClear(); getFocusedWebContents.mockReset(); getFocusedWebContents.mockReturnValue(null); @@ -530,6 +539,233 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("captures hidden preview recordings independently for concurrent tabs", () => + withManager((manager) => + Effect.gen(function* () { + const firstJpeg = Buffer.from("first-recording-frame"); + const secondJpeg = Buffer.from("second-recording-frame"); + const firstCapturePage = vi.fn(async () => ({ + toJPEG: () => firstJpeg, + getSize: () => ({ width: 800, height: 600 }), + })); + const secondCapturePage = vi.fn(async () => ({ + toJPEG: () => secondJpeg, + getSize: () => ({ width: 390, height: 844 }), + })); + const firstSendCommand = vi.fn(async () => undefined); + const secondSendCommand = vi.fn(async () => undefined); + const makeWebContents = ( + id: number, + capturePage: typeof firstCapturePage, + sendCommand: typeof firstSendCommand, + ) => + ({ + id, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => `https://example.com/${id}`, + getTitle: () => `Example ${id}`, + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + }) as never; + const webContentsById = new Map([ + [41, makeWebContents(41, firstCapturePage, firstSendCommand)], + [42, makeWebContents(42, secondCapturePage, secondSendCommand)], + ]); + fromId.mockImplementation((id) => + id === undefined ? null : (webContentsById.get(id) ?? null), + ); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_1"); + yield* manager.createTab("tab_2"); + yield* manager.registerWebview("tab_1", 41); + yield* manager.registerWebview("tab_2", 42); + yield* Effect.all([manager.startRecording("tab_1"), manager.startRecording("tab_2")], { + concurrency: 2, + discard: true, + }); + + expect(firstCapturePage).toHaveBeenCalledOnce(); + expect(secondCapturePage).toHaveBeenCalledOnce(); + expect(frames).toHaveLength(2); + expect(frames).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tabId: "tab_1", + data: firstJpeg.toString("base64"), + width: 800, + height: 600, + }), + expect.objectContaining({ + tabId: "tab_2", + data: secondJpeg.toString("base64"), + width: 390, + height: 844, + }), + ]), + ); + expect(firstSendCommand).not.toHaveBeenCalledWith( + "Page.startScreencast", + expect.anything(), + ); + expect(secondSendCommand).not.toHaveBeenCalledWith( + "Page.startScreencast", + expect.anything(), + ); + + yield* Effect.all([manager.stopRecording("tab_1"), manager.stopRecording("tab_2")], { + concurrency: 2, + discard: true, + }); + }), + ), + ); + + effectIt.effect("shares background frame capture between recording and picture-in-picture", () => + withManager((manager) => + Effect.gen(function* () { + const jpeg = Buffer.from("shared-preview-frame"); + const capturePage = vi.fn(async () => ({ + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand: vi.fn(async () => undefined), + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + } as never); + + const pictureInPictureListeners = new Map void>(); + const pictureInPictureSend = vi.fn(); + const pictureInPictureWindow = { + isDestroyed: vi.fn(() => false), + once: vi.fn((event: string, listener: () => void) => { + pictureInPictureListeners.set(event, listener); + }), + setAlwaysOnTop: vi.fn(), + setVisibleOnAllWorkspaces: vi.fn(), + setAspectRatio: vi.fn(), + loadURL: vi.fn(async () => undefined), + showInactive: vi.fn(), + close: vi.fn(() => { + pictureInPictureListeners.get("closed")?.(); + }), + webContents: { + send: pictureInPictureSend, + }, + }; + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const states: PreviewManager.PreviewTabState[] = []; + const recordingFrames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + recordingFrames.push(frame); + }), + ); + yield* manager.createTab("tab_pip"); + yield* manager.registerWebview("tab_pip", 42); + yield* manager.openPictureInPicture("tab_pip"); + + expect(browserWindowConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + alwaysOnTop: true, + show: false, + skipTaskbar: true, + webPreferences: expect.objectContaining({ + preload: "/tmp/t3/desktop/preview-pip-preload.cjs", + backgroundThrottling: false, + }), + }), + ); + expect(pictureInPictureWindow.showInactive).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.setAspectRatio).toHaveBeenCalledWith(1280 / 720); + expect(pictureInPictureSend).toHaveBeenCalledWith( + "desktop:preview-pip-frame", + expect.objectContaining({ + tabId: "tab_pip", + data: jpeg.toString("base64"), + width: 1280, + height: 720, + }), + ); + expect(states.at(-1)?.pictureInPicture).toBe(true); + expect(capturePage).toHaveBeenCalledOnce(); + + yield* manager.startRecording("tab_pip"); + expect(capturePage).toHaveBeenCalledOnce(); + expect(recordingFrames).toHaveLength(0); + + yield* TestClock.adjust(100); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(recordingFrames).toHaveLength(1); + + yield* manager.stopRecording("tab_pip"); + const framesBeforePictureInPictureOnlyTick = pictureInPictureSend.mock.calls.length; + yield* TestClock.adjust(100); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(pictureInPictureSend.mock.calls.length).toBeGreaterThan( + framesBeforePictureInPictureOnlyTick, + ); + expect(recordingFrames).toHaveLength(1); + + yield* manager.closePictureInPicture("tab_pip"); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + const capturesAfterClose = capturePage.mock.calls.length; + yield* TestClock.adjust(200); + expect(capturePage).toHaveBeenCalledTimes(capturesAfterClose); + }), + ), + ); + effectIt.effect("keeps element picking active during subframe navigation", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 2d0360ef72c..09cf3abfb4f 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -27,14 +27,7 @@ import type { } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; -import { - type BrowserWindow, - type Session, - clipboard, - nativeImage, - shell, - webContents, -} from "electron"; +import { BrowserWindow, type Session, clipboard, nativeImage, shell, webContents } from "electron"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; @@ -52,6 +45,7 @@ import * as Scope from "effect/Scope"; import * as SynchronizedRef from "effect/SynchronizedRef"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; +import { PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "../ipc/channels.ts"; import * as BrowserSession from "./BrowserSession.ts"; import { ANNOTATION_CAPTURED_CHANNEL, @@ -84,6 +78,7 @@ export interface PreviewTabState { canGoBack: boolean; canGoForward: boolean; zoomFactor: number; + pictureInPicture: boolean; controller: "human" | "agent" | "none"; updatedAt: string; } @@ -99,6 +94,12 @@ const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_INTERACTIVE_ELEMENTS = 200; const MAX_SCREENSHOT_WIDTH = 1280; +const RECORDING_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12); +const RECORDING_JPEG_QUALITY = 80; +const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; +const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320; +const PICTURE_IN_PICTURE_MIN_WIDTH = 240; +const PICTURE_IN_PICTURE_MIN_HEIGHT = 160; const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; @@ -124,6 +125,35 @@ const DEFAULT_ANNOTATION_THEME: DesktopPreviewAnnotationTheme = { fontMono: "ui-monospace, monospace", }; +export const buildPreviewPictureInPictureDataUrl = (): string => { + const html = ` + + + + + + + + + Live browser preview + + +`; + return `data:text/html;charset=utf-8,${encodeURIComponent(html)}`; +}; + const artifactSiteSlug = (rawUrl: string): string => { try { const url = new URL(rawUrl); @@ -294,6 +324,13 @@ interface ManagedListeners { readonly scope: Scope.Closeable; } +type FrameCaptureConsumer = "picture-in-picture" | "recording"; + +interface FrameCaptureSession { + readonly scope: Scope.Closeable; + readonly consumers: ReadonlySet; +} + interface PickSession { readonly cancel: Effect.Effect; } @@ -378,6 +415,7 @@ const inputSignalsMatch = (left: PreviewInputSignal, right: PreviewInputSignal): const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function* ( artifactDirectory: string, + pictureInPicturePreloadPath: string, ) { const fileSystem = yield* FileSystem.FileSystem; const hostPlatform = yield* HostProcessPlatform; @@ -413,7 +451,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function >(new Map()); const actionSequenceRef = yield* Ref.make(0); const pointerSequenceRef = yield* Ref.make(0); - const recordingTabIdRef = yield* Ref.make>(Option.none()); + const frameCaptureSessionsRef = yield* SynchronizedRef.make< + ReadonlyMap + >(new Map()); + const pictureInPictureWindowsRef = yield* SynchronizedRef.make< + ReadonlyMap + >(new Map()); + const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); const attempt = (errorContext: PreviewOperationContext, evaluate: () => A) => Effect.try({ @@ -444,6 +488,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function update(copy); return copy; }; + const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* ( + tabId: string, + consumer: FrameCaptureConsumer, + ) { + const captureScope = yield* SynchronizedRef.modify(frameCaptureSessionsRef, (sessions) => { + const current = sessions.get(tabId); + if (!current || !current.consumers.has(consumer)) { + return [undefined, sessions] as const; + } + const consumers = new Set(current.consumers); + consumers.delete(consumer); + if (consumers.size > 0) { + return [ + undefined, + replaceMap(sessions, (copy) => { + copy.set(tabId, { ...current, consumers }); + }), + ] as const; + } + return [ + current.scope, + replaceMap(sessions, (copy) => { + copy.delete(tabId); + }), + ] as const; + }); + if (captureScope) { + yield* Scope.close(captureScope, Exit.void).pipe(Effect.ignore); + } + }); const deliverEvent = ( eventKind: "state-change" | "recording-frame" | "pointer-event", @@ -537,15 +611,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ); - const tabIdForWebContents = Effect.fn("PreviewManager.tabIdForWebContents")(function* ( - webContentsId: number, - ) { - const tabs = yield* SynchronizedRef.get(tabsRef); - return ( - Array.from(tabs.entries()).find(([, tab]) => tab.webContentsId === webContentsId)?.[0] ?? null - ); - }); - const pushBounded = (buffer: ReadonlyArray, entry: A): ReadonlyArray => [...buffer, entry].slice(-DIAGNOSTIC_BUFFER_LIMIT); @@ -729,42 +794,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const scope = yield* Scope.fork(parentScope, "sequential"); const handleDebuggerMessage = Effect.fn("PreviewManager.handleDebuggerMessage")( function* (method: string, params: Record) { - if (method === "Page.screencastFrame") { - const sessionId = params["sessionId"]; - if (typeof sessionId === "number") { - yield* attemptPromise( - { - operation: "ackScreencastFrame", - webContentsId: wc.id, - }, - () => wc.debugger.sendCommand("Page.screencastFrameAck", { sessionId }), - ).pipe(Effect.ignore); - } - const tabId = yield* tabIdForWebContents(wc.id); - const metadata = - typeof params["metadata"] === "object" && params["metadata"] !== null - ? (params["metadata"] as Record) - : {}; - if (tabId && typeof params["data"] === "string") { - const receivedAt = yield* currentIso; - const listeners = yield* Ref.get(recordingFrameListenersRef); - const frame: DesktopPreviewRecordingFrame = { - tabId, - data: params["data"], - width: - typeof metadata["deviceWidth"] === "number" ? metadata["deviceWidth"] : 0, - height: - typeof metadata["deviceHeight"] === "number" ? metadata["deviceHeight"] : 0, - receivedAt, - }; - yield* Effect.forEach( - listeners, - (listener) => - deliverEvent("recording-frame", frame.tabId, () => listener(frame)), - { discard: true }, - ); - } - } yield* captureDiagnosticMessage(wc.id, method, params); }, ); @@ -1274,6 +1303,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function window: BrowserWindow, ) { yield* Ref.set(mainWindowRef, Option.some(window)); + window.once("closed", () => { + runFork(closeAllPictureInPicture()); + }); }); const createTab = Effect.fn("PreviewManager.createTab")(function* (tabId: string) { @@ -1288,6 +1320,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function canGoBack: false, canGoForward: false, zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, controller: "none", updatedAt, }; @@ -1305,7 +1338,17 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); if (!tab) return; - yield* cancelPickElement(tabId); + yield* Effect.all( + [ + cancelPickElement(tabId), + closePictureInPicture(tabId), + stopFrameCapture(tabId, "recording"), + ], + { + concurrency: 3, + discard: true, + }, + ); if (tab.webContentsId != null) { yield* Effect.all( [detachControlSession(tab.webContentsId), detachListeners(tab.webContentsId)], @@ -1320,6 +1363,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function canGoBack: false, canGoForward: false, zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, controller: "none", updatedAt, }; @@ -1457,6 +1501,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function canGoBack: current?.canGoBack ?? false, canGoForward: current?.canGoForward ?? false, zoomFactor: current?.zoomFactor ?? DEFAULT_ZOOM_FACTOR, + pictureInPicture: current?.pictureInPicture ?? false, controller: current?.controller ?? "none", updatedAt, }; @@ -1737,40 +1782,311 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); - const startScreencast = Effect.fn("PreviewManager.startScreencast")(function* ( - send: SendCommand, + const capturePreviewFrame = Effect.fn("PreviewManager.capturePreviewFrame")(function* ( + tabId: string, ) { - yield* send("Page.enable"); - yield* send("Page.startScreencast", { - format: "jpeg", - quality: 80, - maxWidth: 1600, - maxHeight: 1200, - everyNthFrame: 1, - }); + const captureSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (!captureSession) return; + const wc = yield* requireWebContents(tabId); + const image = yield* attemptPromise( + { + operation: "frameCapture.capturePage", + tabId, + webContentsId: wc.id, + }, + () => wc.capturePage(), + ); + const encoded = yield* attempt( + { + operation: "frameCapture.encodeFrame", + tabId, + webContentsId: wc.id, + }, + () => ({ + data: image.toJPEG(RECORDING_JPEG_QUALITY).toString("base64"), + size: image.getSize(), + }), + ); + const receivedAt = yield* currentIso; + const frame: DesktopPreviewRecordingFrame = { + tabId, + data: encoded.data, + width: encoded.size.width, + height: encoded.size.height, + receivedAt, + }; + const deliveries: Array> = []; + if (captureSession.consumers.has("recording")) { + const listeners = yield* Ref.get(recordingFrameListenersRef); + deliveries.push( + Effect.forEach( + listeners, + (listener) => deliverEvent("recording-frame", frame.tabId, () => listener(frame)), + { discard: true }, + ), + ); + } + if (captureSession.consumers.has("picture-in-picture")) { + const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get( + tabId, + ); + if (pictureInPictureWindow && !pictureInPictureWindow.isDestroyed()) { + deliveries.push( + Effect.gen(function* () { + const previousAspectRatio = (yield* Ref.get(pictureInPictureAspectRatiosRef)).get( + tabId, + ); + const aspectRatio = frame.width / frame.height; + if (previousAspectRatio !== aspectRatio) { + yield* attempt( + { + operation: "pictureInPicture.setAspectRatio", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureWindow.setAspectRatio(aspectRatio), + ); + yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => + replaceMap(aspectRatios, (copy) => { + copy.set(tabId, aspectRatio); + }), + ); + } + yield* attempt( + { + operation: "pictureInPicture.deliverFrame", + tabId, + webContentsId: wc.id, + }, + () => { + pictureInPictureWindow.webContents.send( + PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, + frame, + ); + }, + ); + }).pipe( + Effect.catch((error) => + Effect.logWarning("Picture-in-picture frame delivery failed.", { + tabId, + error, + }), + ), + ), + ); + } + } + yield* Effect.all(deliveries, { concurrency: 2, discard: true }); }); - const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { - const recordingTabId = yield* Ref.get(recordingTabIdRef); - if (Option.isSome(recordingTabId) && recordingTabId.value !== tabId) { - return yield* new PreviewRecordingAlreadyActiveError({ - requestedTabId: tabId, - activeTabId: recordingTabId.value, + const startFrameCapture = Effect.fn("PreviewManager.startFrameCapture")(function* ( + tabId: string, + consumer: FrameCaptureConsumer, + ) { + const created = yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => { + const current = sessions.get(tabId); + if (current) { + if (current.consumers.has(consumer)) { + return Effect.succeed([false, sessions] as const); + } + return Effect.succeed([ + false, + replaceMap(sessions, (copy) => { + copy.set(tabId, { + ...current, + consumers: new Set([...current.consumers, consumer]), + }); + }), + ] as const); + } + return Effect.gen(function* () { + const scope = yield* Scope.fork(parentScope, "sequential"); + return [ + true, + replaceMap(sessions, (copy) => { + copy.set(tabId, { + scope, + consumers: new Set([consumer]), + }); + }), + ] as const; }); + }); + if (!created) return; + const session = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (!session) return; + const captureNextFrame = Effect.sleep(RECORDING_FRAME_INTERVAL_MS).pipe( + Effect.andThen(capturePreviewFrame(tabId)), + Effect.catch((error) => + Effect.logWarning("Background preview frame capture failed.", { + tabId, + error, + }), + ), + ); + yield* Effect.forkIn(Effect.forever(captureNextFrame), session.scope); + yield* capturePreviewFrame(tabId).pipe(Effect.onError(() => stopFrameCapture(tabId, consumer))); + }); + + const releasePictureInPicture = Effect.fn("PreviewManager.releasePictureInPicture")(function* ( + tabId: string, + expectedWindow: BrowserWindow, + closeWindow: boolean, + ) { + const removed = yield* SynchronizedRef.modify(pictureInPictureWindowsRef, (windows) => { + if (windows.get(tabId) !== expectedWindow) { + return [false, windows] as const; + } + return [ + true, + replaceMap(windows, (copy) => { + copy.delete(tabId); + }), + ] as const; + }); + if (!removed) return; + yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => + replaceMap(aspectRatios, (copy) => { + copy.delete(tabId); + }), + ); + yield* stopFrameCapture(tabId, "picture-in-picture"); + const tabs = yield* SynchronizedRef.get(tabsRef); + if (tabs.has(tabId)) { + yield* update(tabId, { pictureInPicture: false }); + } + if (closeWindow && !expectedWindow.isDestroyed()) { + yield* attempt({ operation: "pictureInPicture.close", tabId }, () => + expectedWindow.close(), + ).pipe(Effect.ignore); } - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "recording.start", startScreencast); - yield* Ref.set(recordingTabIdRef, Option.some(tabId)); }); - const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) { - const recordingTabId = yield* Ref.get(recordingTabIdRef); - if (Option.isNone(recordingTabId) || recordingTabId.value !== tabId) return; + const closePictureInPicture = Effect.fn("PreviewManager.closePictureInPicture")(function* ( + tabId: string, + ) { + const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get( + tabId, + ); + if (!pictureInPictureWindow) { + yield* stopFrameCapture(tabId, "picture-in-picture"); + return; + } + yield* releasePictureInPicture(tabId, pictureInPictureWindow, true); + }); + + const closeAllPictureInPicture = Effect.fn("PreviewManager.closeAllPictureInPicture")( + function* () { + const windows = yield* SynchronizedRef.get(pictureInPictureWindowsRef); + yield* Effect.forEach(windows.keys(), closePictureInPicture, { + concurrency: "unbounded", + discard: true, + }); + }, + ); + + const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( + tabId: string, + ) { const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "recording.stop", (send) => - send("Page.stopScreencast").pipe(Effect.asVoid), + const existing = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get(tabId); + if (existing && !existing.isDestroyed()) { + yield* attempt({ operation: "pictureInPicture.showExisting", tabId }, () => + existing.showInactive(), + ); + return; + } + const title = wc.getTitle().trim(); + const pictureInPictureWindow = yield* attempt( + { + operation: "pictureInPicture.create", + tabId, + webContentsId: wc.id, + }, + () => + new BrowserWindow({ + width: PICTURE_IN_PICTURE_INITIAL_WIDTH, + height: PICTURE_IN_PICTURE_INITIAL_HEIGHT, + minWidth: PICTURE_IN_PICTURE_MIN_WIDTH, + minHeight: PICTURE_IN_PICTURE_MIN_HEIGHT, + title: title.length > 0 ? `Preview · ${title}` : "Browser preview", + show: false, + alwaysOnTop: true, + autoHideMenuBar: true, + fullscreenable: false, + maximizable: false, + minimizable: false, + resizable: true, + skipTaskbar: true, + backgroundColor: "#111111", + ...(hostPlatform === "darwin" ? { type: "panel" as const } : {}), + webPreferences: { + preload: pictureInPicturePreloadPath, + backgroundThrottling: false, + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }), + ); + const onClosed = () => { + runFork(releasePictureInPicture(tabId, pictureInPictureWindow, false)); + }; + yield* attempt( + { + operation: "pictureInPicture.configure", + tabId, + webContentsId: wc.id, + }, + () => { + pictureInPictureWindow.once("closed", onClosed); + pictureInPictureWindow.setAlwaysOnTop( + true, + hostPlatform === "darwin" ? "floating" : "normal", + ); + if (hostPlatform === "darwin") { + pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { + visibleOnFullScreen: true, + }); + } + }, + ); + yield* SynchronizedRef.update(pictureInPictureWindowsRef, (windows) => + replaceMap(windows, (copy) => { + copy.set(tabId, pictureInPictureWindow); + }), ); - yield* Ref.set(recordingTabIdRef, Option.none()); + const initialize = Effect.gen(function* () { + yield* attemptPromise( + { + operation: "pictureInPicture.load", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureWindow.loadURL(buildPreviewPictureInPictureDataUrl()), + ); + yield* startFrameCapture(tabId, "picture-in-picture"); + yield* attempt( + { + operation: "pictureInPicture.show", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureWindow.showInactive(), + ); + yield* update(tabId, { pictureInPicture: true }); + }); + yield* initialize.pipe( + Effect.onError(() => releasePictureInPicture(tabId, pictureInPictureWindow, true)), + ); + }); + + const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { + yield* startFrameCapture(tabId, "recording"); + }); + + const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) { + yield* stopFrameCapture(tabId, "recording"); }); const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* ( @@ -2518,6 +2834,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function goForward, hardReload, navigate, + openPictureInPicture, openDevTools, pickElement, refresh, @@ -2528,6 +2845,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function setAnnotationTheme, setMainWindow, startRecording, + closePictureInPicture, stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), @@ -2846,6 +3164,8 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly revealArtifact: (path: string) => Effect.Effect; readonly copyArtifactToClipboard: (path: string) => Effect.Effect; + readonly openPictureInPicture: (tabId: string) => Effect.Effect; + readonly closePictureInPicture: (tabId: string) => Effect.Effect; readonly startRecording: (tabId: string) => Effect.Effect; readonly stopRecording: (tabId: string) => Effect.Effect; readonly saveRecording: ( @@ -2896,7 +3216,10 @@ export class PreviewManager extends Context.Service< export const make = Effect.gen(function* PreviewManagerMake() { const environment = yield* DesktopEnvironment.DesktopEnvironment; const browserSession = yield* BrowserSession.BrowserSession; - const operations = yield* makeNativeOperations(environment.browserArtifactsDir); + const operations = yield* makeNativeOperations( + environment.browserArtifactsDir, + environment.path.join(environment.dirname, "preview-pip-preload.cjs"), + ); return PreviewManager.of({ setMainWindow: operations.setMainWindow, @@ -2953,6 +3276,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { captureScreenshot: operations.captureScreenshot, revealArtifact: operations.revealArtifact, copyArtifactToClipboard: operations.copyArtifactToClipboard, + openPictureInPicture: operations.openPictureInPicture, + closePictureInPicture: operations.closePictureInPicture, startRecording: operations.startRecording, stopRecording: operations.stopRecording, saveRecording: operations.saveRecording, diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index 587da8d4431..3cd06cfcf32 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -315,6 +315,7 @@ const makeSplashScenario = (createOutcomes: readonly (Electron.BrowserWindow | n webPreferences: { preload: null, partition: null, + backgroundThrottling: null, sandbox: null, contextIsolation: null, nodeIntegration: null, @@ -429,6 +430,7 @@ describe("DesktopWindow", () => { assert.isUndefined(createdWindowOptions[0]?.x); assert.isUndefined(createdWindowOptions[0]?.y); assert.isTrue(createdWindowOptions[0]?.disableAutoHideCursor); + assert.isFalse(createdWindowOptions[0]?.webPreferences?.backgroundThrottling); assert.deepEqual(fakeWindow.setAutoHideCursor.mock.calls, [[false]]); assert.deepEqual(fakeWindow.loadURL.mock.calls[0], ["t3code-dev://app/"]); assert.equal(fakeWindow.openDevTools.mock.calls.length, 1); diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index db4b698434d..40788009d3b 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -331,6 +331,7 @@ export const make = Effect.gen(function* () { ...getWindowTitleBarOptions(shouldUseDarkColors, environment.platform), webPreferences: { preload: environment.preloadPath, + backgroundThrottling: false, contextIsolation: true, nodeIntegration: false, sandbox: true, diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 96e089b9183..9f25204f163 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -73,5 +73,12 @@ export default defineConfig({ alwaysBundle: (id) => id === "react-grab" || id.startsWith("react-grab/"), }, }, + { + format: "cjs", + outDir: "dist-electron", + sourcemap: true, + outExtensions: () => ({ js: ".cjs" }), + entry: ["src/preview-pip-preload.ts"], + }, ], }); diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts index ffda427d815..5bd665e5179 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts @@ -174,7 +174,7 @@ it.effect("does not let an older response replace a newer explicit tab target", ), ); -it.effect("does not replace the default tab with a globally stopped recording tab", () => +it.effect("tracks the tab returned by a targeted recording stop", () => Effect.scoped( Effect.gen(function* () { const broker = yield* makeBroker; @@ -203,7 +203,7 @@ it.effect("does not replace the default tab with a globally stopped recording ta yield* broker.invoke({ scope, operation: "recordingStop", input: {} }); yield* broker.invoke({ scope, operation: "snapshot", input: {} }); - expect(routedRequests.at(-1)?.tabId).toBe(browsingTabId); + expect(routedRequests.at(-1)?.tabId).toBe(recordingTabId); }), ), ); diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index 7ed77aabdf1..5a952803033 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -548,8 +548,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { }); }); const result = yield* awaitResponse().pipe(Effect.ensuring(removePending)); - // A stop artifact identifies the globally recorded tab, not the caller's browsing target. - const responseTabId = input.operation === "recordingStop" ? undefined : readResultTabId(result); + const responseTabId = readResultTabId(result); const resultTabId = responseTabId === undefined ? input.tabId : responseTabId; if (resultTabId === undefined) return result; const assignmentKey = hostAssignmentKey(input.scope); diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index c729fc20ece..f65cfed0bb4 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -176,7 +176,8 @@ export const PreviewRecordingStartTool = safeBrowserTool( export const PreviewRecordingStopTool = safeBrowserTool( Tool.make("preview_recording_stop", { - description: "Stop the active browser recording and save it as a local evidence artifact.", + description: + "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and save it as a local evidence artifact.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationRecordingArtifact, failure: PreviewAutomationError, diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index a72da49c6fa..4e38f570c35 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -8,7 +8,6 @@ import { previewBridge } from "~/components/preview/previewBridge"; import { usePreviewBridge } from "~/components/preview/usePreviewBridge"; import { cn } from "~/lib/utils"; -import { stopBrowserRecording, useActiveBrowserRecordingTabId } from "./browserRecording"; import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore"; import { browserViewportSettingKey } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; @@ -47,7 +46,6 @@ export function HostedBrowserWebview(props: { const wrapperRef = useRef(null); const webviewRef = useRef(null); const [aspectRatioLocked, setAspectRatioLocked] = useState(false); - const activeRecordingTabId = useActiveBrowserRecordingTabId(); const presentation = useBrowserSurfaceStore( useShallow((state) => { const current = state.byTabId[tabId]; @@ -59,11 +57,6 @@ export function HostedBrowserWebview(props: { ); usePreviewBridge({ threadRef, tabId }); - useEffect(() => { - if (presentation.visible || activeRecordingTabId !== tabId) return; - void stopBrowserRecording(tabId).catch(() => undefined); - }, [activeRecordingTabId, presentation.visible, tabId]); - useEffect(() => { const lease = acquireDesktopTab(tabId); tabLeaseRef.current = lease; diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 9810987c9b9..26fd055eb3e 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -9,12 +9,14 @@ const { events, onFrame, registrySet, save, startScreencast, stopScreencast, sur return { events, onFrame: vi.fn(() => vi.fn()), - registrySet: vi.fn((_atom: unknown, value: string | null) => { - events.push(value === null ? "clear" : `publish:${value}`); + registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { + events.push( + value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, + ); }), - save: vi.fn(async () => ({ + save: vi.fn(async (tabId: string) => ({ id: "recording-test", - tabId: "recording-tab", + tabId, path: "/tmp/recording-test.webm", mimeType: "video/webm" as const, sizeBytes: 0, @@ -48,7 +50,6 @@ import { BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS, BrowserRecordingConflictError, BrowserRecordingOperationError, - BrowserRecordingRequiresVisibleTabError, startBrowserRecording, stopBrowserRecording, } from "./browserRecording"; @@ -116,7 +117,7 @@ describe("browser recording", () => { await stopBrowserRecording("recording-tab"); }); - it("rejects recording for a hidden tab before starting screencast", async () => { + it("records a hidden tab without requiring it to become visible", async () => { surfaceState.byTabId = { "recording-tab": { visible: false, @@ -125,12 +126,38 @@ describe("browser recording", () => { }, }; - await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf( - BrowserRecordingRequiresVisibleTabError, - ); + await startBrowserRecording("recording-tab"); + + expect(startScreencast).toHaveBeenCalledWith("recording-tab"); + expect(events).toEqual(["start-screencast", "publish:recording-tab"]); + + await stopBrowserRecording("recording-tab"); + }); + + it("records separate tabs concurrently", async () => { + surfaceState.byTabId = { + ...surfaceState.byTabId, + "recording-tab-2": { + visible: false, + rect: { x: 0, y: 0, width: 390, height: 844 }, + content: { x: 0, y: 0, width: 390, height: 844, scale: 1, scrollLeft: 0, scrollTop: 0 }, + }, + }; + + await Promise.all([ + startBrowserRecording("recording-tab"), + startBrowserRecording("recording-tab-2"), + ]); + + expect(startScreencast).toHaveBeenCalledTimes(2); + expect(onFrame).toHaveBeenCalledOnce(); + expect(events).toContain("publish:recording-tab,recording-tab-2"); - expect(startScreencast).not.toHaveBeenCalled(); - expect(registrySet).not.toHaveBeenCalled(); + await Promise.all([ + stopBrowserRecording("recording-tab"), + stopBrowserRecording("recording-tab-2"), + ]); + expect(save).toHaveBeenCalledTimes(2); }); it("does not report success for a second start while the first is still starting", async () => { diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 01da1e4a3c1..7d1748113fa 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -46,17 +46,6 @@ export class BrowserRecordingCanvasUnavailableError extends Schema.TaggedErrorCl } } -export class BrowserRecordingRequiresVisibleTabError extends Schema.TaggedErrorClass()( - "BrowserRecordingRequiresVisibleTabError", - { - tabId: Schema.String, - }, -) { - override get message(): string { - return `Browser recording requires tab ${this.tabId} to be visible.`; - } -} - export class BrowserRecordingOperationError extends Schema.TaggedErrorClass()( "BrowserRecordingOperationError", { @@ -102,37 +91,36 @@ interface ActiveRecording { lifecycle: BrowserRecordingLifecycle; } -const activeBrowserRecordingTabIdAtom = Atom.make(null).pipe( - Atom.keepAlive, - Atom.withLabel("preview:active-browser-recording-tab"), -); +interface ActiveBrowserRecordingIndex { + readonly tabIds: ReadonlySet; +} + +const activeBrowserRecordingTabIdsAtom = Atom.make({ + tabIds: new Set(), +}).pipe(Atom.keepAlive, Atom.withLabel("preview:active-browser-recording-tabs")); -export function useActiveBrowserRecordingTabId(): string | null { - return useAtomValue(activeBrowserRecordingTabIdAtom); +export function useActiveBrowserRecordingTabIds(): ReadonlySet { + return useAtomValue(activeBrowserRecordingTabIdsAtom).tabIds; } -let active: ActiveRecording | null = null; +const activeRecordings = new Map(); let unsubscribeFrames: (() => void) | null = null; export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000; -export function readActiveBrowserRecordingTabId(): string | null { - return active?.tabId ?? null; -} - const preferredMimeType = (): string => { const candidates = ["video/mp4;codecs=avc1.42E01E", "video/webm;codecs=vp9", "video/webm"]; return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)) ?? "video/webm"; }; const drawFrame = (frame: DesktopPreviewRecordingFrame): void => { - const recording = active; - if (!recording || recording.tabId !== frame.tabId) return; + const recording = activeRecordings.get(frame.tabId); + if (!recording) return; const image = new Image(); image.addEventListener( "load", () => { - if (active !== recording) return; + if (activeRecordings.get(frame.tabId) !== recording) return; recording.context.drawImage(image, 0, 0, recording.canvas.width, recording.canvas.height); }, { once: true }, @@ -150,11 +138,15 @@ const stopMediaRecorder = async (recorder: MediaRecorder): Promise => { }; const clearActiveRecording = (recording: ActiveRecording): void => { - if (active !== recording) return; - active = null; - unsubscribeFrames?.(); - unsubscribeFrames = null; - appAtomRegistry.set(activeBrowserRecordingTabIdAtom, null); + if (activeRecordings.get(recording.tabId) !== recording) return; + activeRecordings.delete(recording.tabId); + if (activeRecordings.size === 0) { + unsubscribeFrames?.(); + unsubscribeFrames = null; + } + appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { + tabIds: new Set(activeRecordings.keys()), + }); }; const recordingStartupCancelledError = ( @@ -168,7 +160,7 @@ const recordingStartupCancelledError = ( }); const isRecordingStarting = (recording: ActiveRecording): boolean => - active === recording && recording.lifecycle.phase === "starting"; + activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting"; const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise => { let timeout: ReturnType | null = null; @@ -198,17 +190,17 @@ const isStartupWaitTimeout = (error: unknown): error is BrowserRecordingOperatio export async function startBrowserRecording(tabId: string): Promise { const bridge = previewBridge; if (!bridge) throw new BrowserRecordingUnavailableError({ tabId }); - if (active) { - if (active.tabId === tabId && active.lifecycle.phase === "recording") { - return active.startedAt; + const activeRecording = activeRecordings.get(tabId); + if (activeRecording) { + if (activeRecording.lifecycle.phase === "recording") { + return activeRecording.startedAt; } throw new BrowserRecordingConflictError({ requestedTabId: tabId, - activeTabId: active.tabId, + activeTabId: activeRecording.tabId, }); } const surface = useBrowserSurfaceStore.getState().byTabId[tabId]; - if (!surface?.visible) throw new BrowserRecordingRequiresVisibleTabError({ tabId }); const recordingSize = surface?.content ?? surface?.rect; const canvas = document.createElement("canvas"); canvas.width = Math.max(1, recordingSize?.width ?? 1280); @@ -256,7 +248,7 @@ export async function startBrowserRecording(tabId: string): Promise { startupSettled, lifecycle: { phase: "starting" }, }; - active = recording; + activeRecordings.set(tabId, recording); try { try { unsubscribeFrames ??= bridge.recording.onFrame(drawFrame); @@ -324,7 +316,9 @@ export async function startBrowserRecording(tabId: string): Promise { throw recordingStartupCancelledError(recording); } recording.lifecycle = { phase: "recording" }; - appAtomRegistry.set(activeBrowserRecordingTabIdAtom, tabId); + appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { + tabIds: new Set(activeRecordings.keys()), + }); return startedAt; } finally { settleStartup?.(); @@ -434,14 +428,14 @@ export function stopBrowserRecording( tabId: string, ): Promise { const bridge = previewBridge; - const recording = active; - if (!bridge || !recording || recording.tabId !== tabId) return Promise.resolve(null); + const recording = activeRecordings.get(tabId); + if (!bridge || !recording) return Promise.resolve(null); if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; const stopPromise = Promise.resolve() .then(() => finalizeBrowserRecording(bridge, recording)) .catch((error) => { - if (isStartupWaitTimeout(error) && active === recording) { + if (isStartupWaitTimeout(error) && activeRecordings.get(recording.tabId) === recording) { const cleanupAfterStartup = recording.startupSettled.then(() => discardBrowserRecording(bridge, recording), ); diff --git a/apps/web/src/browser/browserRecordingScope.test.ts b/apps/web/src/browser/browserRecordingScope.test.ts deleted file mode 100644 index 6b3adf2219e..00000000000 --- a/apps/web/src/browser/browserRecordingScope.test.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { describe, expect, it } from "vite-plus/test"; - -import { resolveBrowserRecordingStopTarget } from "./browserRecordingScope"; - -describe("resolveBrowserRecordingStopTarget", () => { - it("stops the active recording when no explicit tab was requested", () => { - expect(resolveBrowserRecordingStopTarget("tab-a")).toBe("tab-a"); - expect(resolveBrowserRecordingStopTarget("tab-b")).toBe("tab-b"); - expect(resolveBrowserRecordingStopTarget(null)).toBeNull(); - }); - - it("only stops an explicitly requested tab when it owns the recording", () => { - expect(resolveBrowserRecordingStopTarget("tab-a", "tab-a")).toBe("tab-a"); - expect(resolveBrowserRecordingStopTarget("tab-a", "tab-b")).toBeNull(); - }); -}); diff --git a/apps/web/src/browser/browserRecordingScope.ts b/apps/web/src/browser/browserRecordingScope.ts deleted file mode 100644 index 92f58016aa1..00000000000 --- a/apps/web/src/browser/browserRecordingScope.ts +++ /dev/null @@ -1,7 +0,0 @@ -export function resolveBrowserRecordingStopTarget( - activeTabId: string | null, - requestedTabId?: string, -): string | null { - if (activeTabId === null) return null; - return requestedTabId === undefined || requestedTabId === activeTabId ? activeTabId : null; -} diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 06954b73f37..4e411914d1c 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -29,12 +29,7 @@ import { } from "~/previewStateStore"; import { useRightPanelStore } from "~/rightPanelStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; -import { - readActiveBrowserRecordingTabId, - startBrowserRecording, - stopBrowserRecording, -} from "~/browser/browserRecording"; -import { resolveBrowserRecordingStopTarget } from "~/browser/browserRecordingScope"; +import { startBrowserRecording, stopBrowserRecording } from "~/browser/browserRecording"; import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; import { isElectron } from "~/env"; import { useEnvironments } from "~/state/environments"; @@ -507,12 +502,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; } case "recordingStop": { - const recordingTabId = readActiveBrowserRecordingTabId(); - const stopTabId = resolveBrowserRecordingStopTarget( - recordingTabId, - request.tabIdExplicit ? request.tabId : undefined, - ); - const artifact = stopTabId ? await stopBrowserRecording(stopTabId) : null; + const artifact = tabId ? await stopBrowserRecording(tabId) : null; if (!artifact) { return raisePreviewAutomationHostError( new PreviewAutomationRecordingNotActiveError({ diff --git a/apps/web/src/components/preview/PreviewChromeRow.tsx b/apps/web/src/components/preview/PreviewChromeRow.tsx index f2c3fdfea76..4c7519e71e3 100644 --- a/apps/web/src/components/preview/PreviewChromeRow.tsx +++ b/apps/web/src/components/preview/PreviewChromeRow.tsx @@ -4,6 +4,7 @@ import { Camera, ExternalLink, MousePointerClick, + PictureInPicture2, RotateCw, } from "lucide-react"; import { @@ -40,6 +41,9 @@ interface Props { onCapture?: ((record: boolean) => void) | undefined; captureDisabled?: boolean | undefined; recording?: boolean | undefined; + onPictureInPicture?: (() => void) | undefined; + pictureInPicture?: boolean | undefined; + pictureInPictureDisabled?: boolean | undefined; /** * When provided, renders an annotation-mode toggle button to the right of * the URL input. Pressed while annotation mode is active (button shows in `pressed` @@ -77,6 +81,9 @@ export function PreviewChromeRow({ onCapture, captureDisabled, recording, + onPictureInPicture, + pictureInPicture, + pictureInPictureDisabled, onPickElement, pickActive, pickDisabled, @@ -274,6 +281,30 @@ export function PreviewChromeRow({ ) : null} + {onPictureInPicture ? ( + + + } + > + + + + {pictureInPicture ? "Close picture-in-picture" : "Keep preview on top"} + + + ) : null} {trailingActions} {loadProgress > 0 ? ( diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index c0d503c7599..b1af7bd062d 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -8,6 +8,11 @@ const mocks = vi.hoisted(() => ({ readPreparedConnection: vi.fn(() => ({ httpBaseUrl: "http://172.25.85.75:3773" })), submittedUrl: null as ((url: string) => void) | null, emptyStateUrl: null as ((url: string) => void) | null, + togglePictureInPicture: null as (() => void) | null, + pictureInPicturePressed: false, + openPictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), + closePictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), + pictureInPicture: false, showEmptyState: false, })); @@ -36,10 +41,12 @@ vi.mock("~/previewStateStore", () => ({ activeTabId: "tab-1", desktopByTabId: { "tab-1": { + hasWebContents: true, canGoBack: false, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: mocks.pictureInPicture, controller: "none", }, }, @@ -79,7 +86,7 @@ vi.mock("~/state/use-atom-command", () => ({ vi.mock("~/browser/browserRecording", () => ({ startBrowserRecording: vi.fn(), stopBrowserRecording: vi.fn(), - useActiveBrowserRecordingTabId: () => null, + useActiveBrowserRecordingTabIds: () => new Set(), })); vi.mock("~/browser/browserSurfaceStore", () => ({ @@ -94,12 +101,24 @@ vi.mock("~/components/ui/toast", () => ({ })); vi.mock("./previewBridge", () => ({ - previewBridge: { navigate: mocks.navigate }, + previewBridge: { + navigate: mocks.navigate, + pictureInPicture: { + open: mocks.openPictureInPicture, + close: mocks.closePictureInPicture, + }, + }, })); vi.mock("./PreviewChromeRow", () => ({ - PreviewChromeRow: (props: { onSubmit: (url: string) => void }) => { + PreviewChromeRow: (props: { + onSubmit: (url: string) => void; + onPictureInPicture?: () => void; + pictureInPicture?: boolean; + }) => { mocks.submittedUrl = props.onSubmit; + mocks.togglePictureInPicture = props.onPictureInPicture ?? null; + mocks.pictureInPicturePressed = props.pictureInPicture ?? false; return null; }, })); @@ -127,6 +146,11 @@ describe("PreviewView navigation", () => { mocks.readPreparedConnection.mockClear(); mocks.submittedUrl = null; mocks.emptyStateUrl = null; + mocks.togglePictureInPicture = null; + mocks.pictureInPicturePressed = false; + mocks.openPictureInPicture.mockClear(); + mocks.closePictureInPicture.mockClear(); + mocks.pictureInPicture = false; mocks.showEmptyState = false; }); @@ -191,4 +215,26 @@ describe("PreviewView navigation", () => { "http://172.25.85.75:5173/app?mode=test#top", ); }); + + it("opens and closes picture-in-picture for the active preview tab", async () => { + const props = { + threadRef: { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }, + tabId: "tab-1", + visible: true, + } as const; + + renderToStaticMarkup(); + expect(mocks.pictureInPicturePressed).toBe(false); + mocks.togglePictureInPicture?.(); + await vi.waitFor(() => expect(mocks.openPictureInPicture).toHaveBeenCalledWith("tab-1")); + + mocks.pictureInPicture = true; + renderToStaticMarkup(); + expect(mocks.pictureInPicturePressed).toBe(true); + mocks.togglePictureInPicture?.(); + await vi.waitFor(() => expect(mocks.closePictureInPicture).toHaveBeenCalledWith("tab-1")); + }); }); diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 5f7deeb0fcd..d1156f81b31 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -47,7 +47,7 @@ import { AgentBrowserCursor } from "./AgentBrowserCursor"; import { startBrowserRecording, stopBrowserRecording, - useActiveBrowserRecordingTabId, + useActiveBrowserRecordingTabIds, } from "~/browser/browserRecording"; import { stackedThreadToast, toastManager } from "~/components/ui/toast"; @@ -67,7 +67,7 @@ const localApi = typeof window === "undefined" ? null : ensureLocalApi(); export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, visible }: Props) { const [focusUrlNonce, setFocusUrlNonce] = useState(undefined); const [pickActive, setPickActive] = useState(false); - const activeRecordingTabId = useActiveBrowserRecordingTabId(); + const activeRecordingTabIds = useActiveBrowserRecordingTabIds(); const pickActiveRef = useRef(false); const isMountedRef = useRef(true); const previewState = useThreadPreviewState(threadRef); @@ -227,11 +227,25 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, void localApi.shell.openExternal(url).catch(() => undefined); }, [url]); + const handlePictureInPicture = useCallback(() => { + if (!previewBridge || !tabId) return; + const operation = desktopOverlay?.pictureInPicture + ? previewBridge.pictureInPicture.close + : previewBridge.pictureInPicture.open; + void operation(tabId).catch((error) => { + toastManager.add({ + type: "error", + title: "Unable to update picture-in-picture", + description: error instanceof Error ? error.message : "An error occurred.", + }); + }); + }, [desktopOverlay?.pictureInPicture, tabId]); + const handleCapture = useCallback( (record: boolean) => { if (!previewBridge || !tabId) return; const bridge = previewBridge; - const recordingThisTab = activeRecordingTabId === tabId; + const recordingThisTab = activeRecordingTabIds.has(tabId); if (recordingThisTab) { void stopBrowserRecording(tabId).then( (artifact) => { @@ -325,14 +339,6 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, return; } if (record) { - if (activeRecordingTabId !== null) { - toastManager.add({ - type: "warning", - title: "Another preview is recording", - description: "Stop the active recording before starting a new one.", - }); - return; - } void startBrowserRecording(tabId).catch((error) => { toastManager.add({ type: "error", @@ -472,7 +478,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, }, ); }, - [activeRecordingTabId, tabId], + [activeRecordingTabIds, tabId], ); const handlePickElement = useCallback(() => { @@ -594,7 +600,10 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, onOpenInBrowser={tabId ? handleOpenInBrowser : undefined} onCapture={previewBridge && tabId ? handleCapture : undefined} captureDisabled={!desktopOverlay || isUnreachable} - recording={tabId !== null && activeRecordingTabId === tabId} + recording={tabId !== null && activeRecordingTabIds.has(tabId)} + onPictureInPicture={previewBridge && tabId ? handlePictureInPicture : undefined} + pictureInPicture={desktopOverlay?.pictureInPicture ?? false} + pictureInPictureDisabled={!desktopOverlay?.hasWebContents || isUnreachable} onPickElement={previewBridge && tabId ? handlePickElement : undefined} pickActive={pickActive} // Disable when there's no tab (nothing to pick on) OR the page @@ -608,7 +617,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, previewBridge ? ( { snapshot, }); applyPreviewDesktopState(ref, snapshot.tabId, { + hasWebContents: true, canGoBack: true, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: false, controller: "none", }); const state = readThreadPreviewState(ref); @@ -308,10 +310,12 @@ describe("previewStateStore (single-tab)", () => { applyPreviewServerSnapshot(ref, first); applyPreviewServerSnapshot(ref, second); applyPreviewDesktopState(ref, first.tabId, { + hasWebContents: true, canGoBack: true, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: false, controller: "none", }); setActivePreviewTab(ref, first.tabId); @@ -353,10 +357,12 @@ describe("previewStateStore (single-tab)", () => { applyPreviewServerSnapshot(ref, stale); applyPreviewServerSnapshot(ref, active); applyPreviewDesktopState(ref, stale.tabId, { + hasWebContents: true, canGoBack: false, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: false, controller: "none", }); diff --git a/apps/web/src/previewStateStore.ts b/apps/web/src/previewStateStore.ts index e44a464ed24..611fb4d3cee 100644 --- a/apps/web/src/previewStateStore.ts +++ b/apps/web/src/previewStateStore.ts @@ -18,10 +18,12 @@ import { PREVIEW_RECENT_URL_LIMIT } from "./components/preview/previewConstants" import { appAtomRegistry } from "./rpc/atomRegistry"; export interface DesktopPreviewOverlay { + hasWebContents: boolean; canGoBack: boolean; canGoForward: boolean; loading: boolean; zoomFactor: number; + pictureInPicture: boolean; controller: "human" | "agent" | "none"; } diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 6c853ca1631..a15962db58d 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -518,6 +518,8 @@ export interface DesktopPreviewTabState { canGoForward: boolean; /** Current zoom factor (1.0 = 100%). */ zoomFactor: number; + /** Whether this tab is currently mirrored into a desktop picture-in-picture window. */ + pictureInPicture: boolean; controller: "human" | "agent" | "none"; updatedAt: string; } @@ -554,6 +556,7 @@ export const DesktopPreviewTabStateSchema: Schema.Codec canGoBack: Schema.Boolean, canGoForward: Schema.Boolean, zoomFactor: Schema.Number, + pictureInPicture: Schema.Boolean, controller: Schema.Literals(["human", "agent", "none"]), updatedAt: Schema.String, }); @@ -1059,6 +1062,10 @@ export interface DesktopPreviewBridge { captureScreenshot: (tabId: string) => Promise; revealArtifact: (path: string) => Promise; copyArtifactToClipboard: (path: string) => Promise; + pictureInPicture: { + open: (tabId: string) => Promise; + close: (tabId: string) => Promise; + }; recording: { startScreencast: (tabId: string) => Promise; stopScreencast: (tabId: string) => Promise; From 43c84dc3c35ccf83d16de75634258472b1a5805f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 00:48:52 +0200 Subject: [PATCH 02/26] Fix preview lifecycle review findings Serialize PiP mutations, reject invalid frames, and stop recordings before final tab teardown. Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 157 ++++++++++++ apps/desktop/src/preview/Manager.ts | 224 ++++++++++-------- .../src/browser/desktopTabLifetime.test.ts | 40 +++- apps/web/src/browser/desktopTabLifetime.ts | 10 +- 4 files changed, 331 insertions(+), 100 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index de8e9839220..a6d17a3eae5 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -114,6 +114,68 @@ const withManager = ( return yield* use(manager); }).pipe(Effect.provide(layer), Effect.scoped); +interface TestCapturedPreviewImage { + readonly toJPEG: () => Buffer; + readonly getSize: () => { readonly width: number; readonly height: number }; +} + +const makeTestPreviewWebContents = ( + capturePage: () => Promise, + id = 42, +) => + ({ + id, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand: vi.fn(async () => undefined), + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + }) as never; + +const makeTestPictureInPictureWindow = (loadURL: () => Promise = async () => undefined) => { + const listeners = new Map void>(); + const send = vi.fn(); + let destroyed = false; + const pictureInPictureWindow = { + isDestroyed: vi.fn(() => destroyed), + once: vi.fn((event: string, listener: () => void) => { + listeners.set(event, listener); + }), + setAlwaysOnTop: vi.fn(), + setVisibleOnAllWorkspaces: vi.fn(), + setAspectRatio: vi.fn(), + loadURL: vi.fn(loadURL), + showInactive: vi.fn(() => { + if (destroyed) throw new Error("Picture-in-picture window is closed."); + }), + close: vi.fn(() => { + if (destroyed) return; + destroyed = true; + listeners.get("closed")?.(); + }), + webContents: { + send, + }, + }; + return { pictureInPictureWindow, send }; +}; + describe("PreviewManager", () => { beforeEach(() => { browserWindowConstructor.mockReset(); @@ -839,6 +901,101 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("drops empty frames before picture-in-picture delivery", () => + withManager((manager) => + Effect.gen(function* () { + const validImage: TestCapturedPreviewImage = { + toJPEG: () => Buffer.from("valid-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + }; + const capturePage = vi.fn(async () => validImage); + capturePage.mockResolvedValueOnce({ + toJPEG: () => Buffer.from("empty-preview-frame"), + getSize: () => ({ width: 0, height: 0 }), + }); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_empty_frame"); + yield* manager.registerWebview("tab_empty_frame", 42); + yield* manager.openPictureInPicture("tab_empty_frame"); + + expect(capturePage).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.setAspectRatio).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + + yield* TestClock.adjust(100); + + expect(pictureInPictureWindow.setAspectRatio).toHaveBeenCalledWith(1280 / 720); + expect(send).toHaveBeenCalledOnce(); + yield* manager.closePictureInPicture("tab_empty_frame"); + }), + ), + ); + + effectIt.effect("serializes concurrent picture-in-picture open and close operations", () => + withManager((manager) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("serialized-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + let resolveLoad: (() => void) | undefined; + const { pictureInPictureWindow } = makeTestPictureInPictureWindow( + () => + new Promise((resolve) => { + resolveLoad = resolve; + }), + ); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.createTab("tab_concurrent_pip"); + yield* manager.registerWebview("tab_concurrent_pip", 42); + + const firstOpen = yield* manager + .openPictureInPicture("tab_concurrent_pip") + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const secondOpen = yield* manager + .openPictureInPicture("tab_concurrent_pip") + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const close = yield* manager + .closePictureInPicture("tab_concurrent_pip") + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + + expect(browserWindowConstructor).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.loadURL).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.close).not.toHaveBeenCalled(); + + resolveLoad?.(); + yield* Fiber.join(firstOpen); + yield* Fiber.join(secondOpen); + yield* Fiber.join(close); + + expect(browserWindowConstructor).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.showInactive).toHaveBeenCalledTimes(2); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + const capturesAfterClose = capturePage.mock.calls.length; + yield* TestClock.adjust(200); + expect(capturePage).toHaveBeenCalledTimes(capturesAfterClose); + }), + ), + ); + effectIt.effect("keeps element picking active during subframe navigation", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 271d319a647..40b1772f93a 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -460,6 +460,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ReadonlyMap >(new Map()); const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); + const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); const attempt = (errorContext: PreviewOperationContext, evaluate: () => A) => Effect.try({ @@ -1860,23 +1861,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, () => wc.capturePage(), ); + const size = yield* attempt( + { + operation: "frameCapture.measureFrame", + tabId, + webContentsId: wc.id, + }, + () => image.getSize(), + ); + if ( + !Number.isFinite(size.width) || + !Number.isFinite(size.height) || + size.width <= 0 || + size.height <= 0 + ) { + return; + } const encoded = yield* attempt( { operation: "frameCapture.encodeFrame", tabId, webContentsId: wc.id, }, - () => ({ - data: image.toJPEG(RECORDING_JPEG_QUALITY).toString("base64"), - size: image.getSize(), - }), + () => image.toJPEG(RECORDING_JPEG_QUALITY).toString("base64"), ); const receivedAt = yield* currentIso; const frame: DesktopPreviewRecordingFrame = { tabId, - data: encoded.data, - width: encoded.size.width, - height: encoded.size.height, + data: encoded, + width: size.width, + height: size.height, receivedAt, }; const deliveries: Array> = []; @@ -2026,17 +2040,23 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } }); + const closePictureInPictureUnlocked = Effect.fn("PreviewManager.closePictureInPictureUnlocked")( + function* (tabId: string) { + const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get( + tabId, + ); + if (!pictureInPictureWindow) { + yield* stopFrameCapture(tabId, "picture-in-picture"); + return; + } + yield* releasePictureInPicture(tabId, pictureInPictureWindow, true); + }, + ); + const closePictureInPicture = Effect.fn("PreviewManager.closePictureInPicture")(function* ( tabId: string, ) { - const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get( - tabId, - ); - if (!pictureInPictureWindow) { - yield* stopFrameCapture(tabId, "picture-in-picture"); - return; - } - yield* releasePictureInPicture(tabId, pictureInPictureWindow, true); + yield* pictureInPictureMutationSemaphore.withPermit(closePictureInPictureUnlocked(tabId)); }); const closeAllPictureInPicture = Effect.fn("PreviewManager.closeAllPictureInPicture")( @@ -2049,100 +2069,110 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, ); - const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( - tabId: string, - ) { - const wc = yield* requireWebContents(tabId); - const existing = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get(tabId); - if (existing && !existing.isDestroyed()) { - yield* attempt({ operation: "pictureInPicture.showExisting", tabId }, () => - existing.showInactive(), - ); - return; - } - const title = wc.getTitle().trim(); - const pictureInPictureWindow = yield* attempt( - { - operation: "pictureInPicture.create", - tabId, - webContentsId: wc.id, - }, - () => - new BrowserWindow({ - width: PICTURE_IN_PICTURE_INITIAL_WIDTH, - height: PICTURE_IN_PICTURE_INITIAL_HEIGHT, - minWidth: PICTURE_IN_PICTURE_MIN_WIDTH, - minHeight: PICTURE_IN_PICTURE_MIN_HEIGHT, - title: title.length > 0 ? `Preview · ${title}` : "Browser preview", - show: false, - alwaysOnTop: true, - autoHideMenuBar: true, - fullscreenable: false, - maximizable: false, - minimizable: false, - resizable: true, - skipTaskbar: true, - backgroundColor: "#111111", - ...(hostPlatform === "darwin" ? { type: "panel" as const } : {}), - webPreferences: { - preload: pictureInPicturePreloadPath, - backgroundThrottling: false, - contextIsolation: true, - nodeIntegration: false, - sandbox: true, - }, - }), - ); - const onClosed = () => { - runFork(releasePictureInPicture(tabId, pictureInPictureWindow, false)); - }; - yield* attempt( - { - operation: "pictureInPicture.configure", - tabId, - webContentsId: wc.id, - }, - () => { - pictureInPictureWindow.once("closed", onClosed); - pictureInPictureWindow.setAlwaysOnTop( - true, - hostPlatform === "darwin" ? "floating" : "normal", + const openPictureInPictureUnlocked = Effect.fn("PreviewManager.openPictureInPictureUnlocked")( + function* (tabId: string) { + const wc = yield* requireWebContents(tabId); + const existing = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get(tabId); + if (existing && !existing.isDestroyed()) { + yield* attempt({ operation: "pictureInPicture.showExisting", tabId }, () => + existing.showInactive(), ); - if (hostPlatform === "darwin") { - pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { - visibleOnFullScreen: true, - }); - } - }, - ); - yield* SynchronizedRef.update(pictureInPictureWindowsRef, (windows) => - replaceMap(windows, (copy) => { - copy.set(tabId, pictureInPictureWindow); - }), - ); - const initialize = Effect.gen(function* () { - yield* attemptPromise( + return; + } + const title = wc.getTitle().trim(); + const pictureInPictureWindow = yield* attempt( { - operation: "pictureInPicture.load", + operation: "pictureInPicture.create", tabId, webContentsId: wc.id, }, - () => pictureInPictureWindow.loadURL(buildPreviewPictureInPictureDataUrl()), + () => + new BrowserWindow({ + width: PICTURE_IN_PICTURE_INITIAL_WIDTH, + height: PICTURE_IN_PICTURE_INITIAL_HEIGHT, + minWidth: PICTURE_IN_PICTURE_MIN_WIDTH, + minHeight: PICTURE_IN_PICTURE_MIN_HEIGHT, + title: title.length > 0 ? `Preview · ${title}` : "Browser preview", + show: false, + alwaysOnTop: true, + autoHideMenuBar: true, + fullscreenable: false, + maximizable: false, + minimizable: false, + resizable: true, + skipTaskbar: true, + backgroundColor: "#111111", + ...(hostPlatform === "darwin" ? { type: "panel" as const } : {}), + webPreferences: { + preload: pictureInPicturePreloadPath, + backgroundThrottling: false, + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }), ); - yield* startFrameCapture(tabId, "picture-in-picture"); + const onClosed = () => { + runFork( + pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, pictureInPictureWindow, false), + ), + ); + }; yield* attempt( { - operation: "pictureInPicture.show", + operation: "pictureInPicture.configure", tabId, webContentsId: wc.id, }, - () => pictureInPictureWindow.showInactive(), + () => { + pictureInPictureWindow.once("closed", onClosed); + pictureInPictureWindow.setAlwaysOnTop( + true, + hostPlatform === "darwin" ? "floating" : "normal", + ); + if (hostPlatform === "darwin") { + pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { + visibleOnFullScreen: true, + }); + } + }, ); - yield* update(tabId, { pictureInPicture: true }); - }); - yield* initialize.pipe( - Effect.onError(() => releasePictureInPicture(tabId, pictureInPictureWindow, true)), - ); + yield* SynchronizedRef.update(pictureInPictureWindowsRef, (windows) => + replaceMap(windows, (copy) => { + copy.set(tabId, pictureInPictureWindow); + }), + ); + const initialize = Effect.gen(function* () { + yield* attemptPromise( + { + operation: "pictureInPicture.load", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureWindow.loadURL(buildPreviewPictureInPictureDataUrl()), + ); + yield* startFrameCapture(tabId, "picture-in-picture"); + yield* attempt( + { + operation: "pictureInPicture.show", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureWindow.showInactive(), + ); + yield* update(tabId, { pictureInPicture: true }); + }); + yield* initialize.pipe( + Effect.onError(() => releasePictureInPicture(tabId, pictureInPictureWindow, true)), + ); + }, + ); + + const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( + tabId: string, + ) { + yield* pictureInPictureMutationSemaphore.withPermit(openPictureInPictureUnlocked(tabId)); }); const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { diff --git a/apps/web/src/browser/desktopTabLifetime.test.ts b/apps/web/src/browser/desktopTabLifetime.test.ts index 1e3b1632bcc..2e39983b134 100644 --- a/apps/web/src/browser/desktopTabLifetime.test.ts +++ b/apps/web/src/browser/desktopTabLifetime.test.ts @@ -1,20 +1,32 @@ -import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -const { closeTab, createTab } = vi.hoisted(() => ({ +const { closeTab, createTab, stopBrowserRecording } = vi.hoisted(() => ({ closeTab: vi.fn(async () => undefined), createTab: vi.fn<() => Promise>(), + stopBrowserRecording: vi.fn(async () => null), })); vi.mock("~/components/preview/previewBridge", () => ({ previewBridge: { closeTab, createTab }, })); +vi.mock("./browserRecording", () => ({ + stopBrowserRecording, +})); + import { acquireDesktopTab } from "./desktopTabLifetime"; describe("desktopTabLifetime", () => { beforeEach(() => { closeTab.mockClear(); createTab.mockClear(); + stopBrowserRecording.mockClear(); + vi.stubGlobal("window", globalThis); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); }); it("shares tab creation readiness across concurrent leases", async () => { @@ -42,4 +54,28 @@ describe("desktopTabLifetime", () => { await first.ready; expect(ready).toBe(true); }); + + it("stops recording before closing the final desktop tab lease", async () => { + vi.useFakeTimers(); + let resolveStop: (() => void) | undefined; + stopBrowserRecording.mockReturnValueOnce( + new Promise((resolve) => { + resolveStop = () => resolve(null); + }), + ); + createTab.mockResolvedValueOnce(undefined); + + const lease = acquireDesktopTab("tab_recording_cleanup"); + await lease.ready; + lease.release(); + await vi.advanceTimersByTimeAsync(0); + + expect(stopBrowserRecording).toHaveBeenCalledWith("tab_recording_cleanup"); + expect(closeTab).not.toHaveBeenCalled(); + + resolveStop?.(); + await Promise.resolve(); + await Promise.resolve(); + expect(closeTab).toHaveBeenCalledWith("tab_recording_cleanup"); + }); }); diff --git a/apps/web/src/browser/desktopTabLifetime.ts b/apps/web/src/browser/desktopTabLifetime.ts index d621f6dc30c..ff8f2cb2676 100644 --- a/apps/web/src/browser/desktopTabLifetime.ts +++ b/apps/web/src/browser/desktopTabLifetime.ts @@ -1,5 +1,7 @@ import { previewBridge } from "~/components/preview/previewBridge"; +import { stopBrowserRecording } from "./browserRecording"; + interface DesktopTabLease { references: number; closeTimer: number | null; @@ -8,6 +10,12 @@ interface DesktopTabLease { const leases = new Map(); +const closeReleasedDesktopTab = async (tabId: string): Promise => { + await stopBrowserRecording(tabId).catch(() => null); + if (leases.has(tabId)) return; + await previewBridge?.closeTab(tabId); +}; + export interface AcquiredDesktopTab { readonly ready: Promise; readonly release: () => void; @@ -37,7 +45,7 @@ export function acquireDesktopTab(tabId: string): AcquiredDesktopTab { const latest = leases.get(tabId); if (!latest || latest.references > 0) return; leases.delete(tabId); - void previewBridge?.closeTab(tabId); + void closeReleasedDesktopTab(tabId).catch(() => undefined); }, 0); }, }; From 6678c487b2c63a6d2b3386815fa53baeba50f828 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 00:54:17 +0200 Subject: [PATCH 03/26] fix(web): serialize desktop tab lifecycle Co-authored-by: codex --- .../src/browser/desktopTabLifetime.test.ts | 27 ++++++++++++++++- apps/web/src/browser/desktopTabLifetime.ts | 29 +++++++++++++++---- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/apps/web/src/browser/desktopTabLifetime.test.ts b/apps/web/src/browser/desktopTabLifetime.test.ts index 2e39983b134..8f7b2ca7d65 100644 --- a/apps/web/src/browser/desktopTabLifetime.test.ts +++ b/apps/web/src/browser/desktopTabLifetime.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; const { closeTab, createTab, stopBrowserRecording } = vi.hoisted(() => ({ - closeTab: vi.fn(async () => undefined), + closeTab: vi.fn<(tabId: string) => Promise>(async () => undefined), createTab: vi.fn<() => Promise>(), stopBrowserRecording: vi.fn(async () => null), })); @@ -78,4 +78,29 @@ describe("desktopTabLifetime", () => { await Promise.resolve(); expect(closeTab).toHaveBeenCalledWith("tab_recording_cleanup"); }); + + it("waits for an in-flight close before recreating a reacquired tab", async () => { + vi.useFakeTimers(); + let resolveClose: (() => void) | undefined; + createTab.mockResolvedValue(undefined); + closeTab.mockReturnValueOnce( + new Promise((resolve) => { + resolveClose = resolve; + }), + ); + + const initial = acquireDesktopTab("tab_close_reacquire"); + await initial.ready; + initial.release(); + await vi.advanceTimersByTimeAsync(0); + + expect(closeTab).toHaveBeenCalledWith("tab_close_reacquire"); + + const reacquired = acquireDesktopTab("tab_close_reacquire"); + expect(createTab).toHaveBeenCalledTimes(1); + + resolveClose?.(); + await reacquired.ready; + expect(createTab).toHaveBeenCalledTimes(2); + }); }); diff --git a/apps/web/src/browser/desktopTabLifetime.ts b/apps/web/src/browser/desktopTabLifetime.ts index ff8f2cb2676..98dffda0ea0 100644 --- a/apps/web/src/browser/desktopTabLifetime.ts +++ b/apps/web/src/browser/desktopTabLifetime.ts @@ -9,11 +9,25 @@ interface DesktopTabLease { } const leases = new Map(); +const pendingTabOperations = new Map>(); -const closeReleasedDesktopTab = async (tabId: string): Promise => { - await stopBrowserRecording(tabId).catch(() => null); - if (leases.has(tabId)) return; - await previewBridge?.closeTab(tabId); +const enqueueDesktopTabOperation = ( + tabId: string, + operation: () => Promise | void, +): Promise => { + const previous = pendingTabOperations.get(tabId); + const pending = previous + ? previous.catch(() => undefined).then(operation) + : Promise.resolve(operation()); + pendingTabOperations.set(tabId, pending); + void pending + .finally(() => { + if (pendingTabOperations.get(tabId) === pending) { + pendingTabOperations.delete(tabId); + } + }) + .catch(() => undefined); + return pending; }; export interface AcquiredDesktopTab { @@ -27,7 +41,7 @@ export function acquireDesktopTab(tabId: string): AcquiredDesktopTab { ({ references: 0, closeTimer: null, - ready: previewBridge?.createTab(tabId) ?? Promise.resolve(), + ready: enqueueDesktopTabOperation(tabId, () => previewBridge?.createTab(tabId)), } satisfies DesktopTabLease); if (current.closeTimer !== null) window.clearTimeout(current.closeTimer); current.references += 1; @@ -45,7 +59,10 @@ export function acquireDesktopTab(tabId: string): AcquiredDesktopTab { const latest = leases.get(tabId); if (!latest || latest.references > 0) return; leases.delete(tabId); - void closeReleasedDesktopTab(tabId).catch(() => undefined); + void enqueueDesktopTabOperation(tabId, async () => { + await stopBrowserRecording(tabId).catch(() => null); + await previewBridge?.closeTab(tabId); + }).catch(() => undefined); }, 0); }, }; From 3a37abc7f19e2dc906253aa5593c23f7af2cc447 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 01:04:21 +0200 Subject: [PATCH 04/26] fix(desktop): make PiP initialization cancellable Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 44 ++-- apps/desktop/src/preview/Manager.ts | 257 +++++++++++++---------- 2 files changed, 179 insertions(+), 122 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index a6d17a3eae5..e20e9c7f944 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -936,7 +936,7 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("serializes concurrent picture-in-picture open and close operations", () => + effectIt.effect("closes an initializing picture-in-picture without blocking later opens", () => withManager((manager) => Effect.gen(function* () { const capturePage = vi.fn(async () => ({ @@ -944,16 +944,20 @@ describe("PreviewManager", () => { getSize: () => ({ width: 1280, height: 720 }), })); fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); - let resolveLoad: (() => void) | undefined; - const { pictureInPictureWindow } = makeTestPictureInPictureWindow( + const { pictureInPictureWindow: initializingWindow } = makeTestPictureInPictureWindow( () => - new Promise((resolve) => { - resolveLoad = resolve; + new Promise(() => { + // Simulate a renderer load that never settles. }), ); - browserWindowConstructor.mockImplementation(function () { - return pictureInPictureWindow; - }); + const { pictureInPictureWindow: reopenedWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor + .mockImplementationOnce(function () { + return initializingWindow; + }) + .mockImplementationOnce(function () { + return reopenedWindow; + }); const states: PreviewManager.PreviewTabState[] = []; yield* manager.subscribeStateChanges((_tabId, state) => Effect.sync(() => { @@ -977,17 +981,27 @@ describe("PreviewManager", () => { yield* Effect.yieldNow; expect(browserWindowConstructor).toHaveBeenCalledOnce(); - expect(pictureInPictureWindow.loadURL).toHaveBeenCalledOnce(); - expect(pictureInPictureWindow.close).not.toHaveBeenCalled(); - - resolveLoad?.(); + expect(initializingWindow.loadURL).toHaveBeenCalledOnce(); + expect(initializingWindow.close).toHaveBeenCalledOnce(); yield* Fiber.join(firstOpen); yield* Fiber.join(secondOpen); yield* Fiber.join(close); - expect(browserWindowConstructor).toHaveBeenCalledOnce(); - expect(pictureInPictureWindow.showInactive).toHaveBeenCalledTimes(2); - expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(initializingWindow.showInactive).toHaveBeenCalledOnce(); + expect(capturePage).not.toHaveBeenCalled(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + + yield* manager.openPictureInPicture("tab_concurrent_pip"); + + expect(browserWindowConstructor).toHaveBeenCalledTimes(2); + expect(reopenedWindow.showInactive).toHaveBeenCalledOnce(); + expect(capturePage).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(true); + + yield* manager.closePictureInPicture("tab_concurrent_pip"); + + expect(browserWindowConstructor).toHaveBeenCalledTimes(2); + expect(reopenedWindow.close).toHaveBeenCalledOnce(); expect(states.at(-1)?.pictureInPicture).toBe(false); const capturesAfterClose = capturePage.mock.calls.length; yield* TestClock.adjust(200); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 40b1772f93a..6d00534e515 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -36,6 +36,7 @@ import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -333,6 +334,11 @@ interface FrameCaptureSession { readonly consumers: ReadonlySet; } +interface PictureInPictureSession { + readonly window: BrowserWindow; + readonly initializationScope: Scope.Closeable; +} + interface PickSession { readonly cancel: Effect.Effect; } @@ -456,8 +462,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const frameCaptureSessionsRef = yield* SynchronizedRef.make< ReadonlyMap >(new Map()); - const pictureInPictureWindowsRef = yield* SynchronizedRef.make< - ReadonlyMap + const pictureInPictureSessionsRef = yield* SynchronizedRef.make< + ReadonlyMap >(new Map()); const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); @@ -1905,9 +1911,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); } if (captureSession.consumers.has("picture-in-picture")) { - const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get( + const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( tabId, - ); + )?.window; if (pictureInPictureWindow && !pictureInPictureWindow.isDestroyed()) { deliveries.push( Effect.gen(function* () { @@ -2008,21 +2014,22 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const releasePictureInPicture = Effect.fn("PreviewManager.releasePictureInPicture")(function* ( tabId: string, - expectedWindow: BrowserWindow, + expectedSession: PictureInPictureSession, closeWindow: boolean, ) { - const removed = yield* SynchronizedRef.modify(pictureInPictureWindowsRef, (windows) => { - if (windows.get(tabId) !== expectedWindow) { - return [false, windows] as const; + const removed = yield* SynchronizedRef.modify(pictureInPictureSessionsRef, (sessions) => { + if (sessions.get(tabId) !== expectedSession) { + return [false, sessions] as const; } return [ true, - replaceMap(windows, (copy) => { + replaceMap(sessions, (copy) => { copy.delete(tabId); }), ] as const; }); if (!removed) return; + yield* Scope.close(expectedSession.initializationScope, Exit.void).pipe(Effect.ignore); yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => replaceMap(aspectRatios, (copy) => { copy.delete(tabId); @@ -2033,23 +2040,23 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (tabs.has(tabId)) { yield* update(tabId, { pictureInPicture: false }); } - if (closeWindow && !expectedWindow.isDestroyed()) { + if (closeWindow && !expectedSession.window.isDestroyed()) { yield* attempt({ operation: "pictureInPicture.close", tabId }, () => - expectedWindow.close(), + expectedSession.window.close(), ).pipe(Effect.ignore); } }); const closePictureInPictureUnlocked = Effect.fn("PreviewManager.closePictureInPictureUnlocked")( function* (tabId: string) { - const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get( + const pictureInPictureSession = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( tabId, ); - if (!pictureInPictureWindow) { + if (!pictureInPictureSession) { yield* stopFrameCapture(tabId, "picture-in-picture"); return; } - yield* releasePictureInPicture(tabId, pictureInPictureWindow, true); + yield* releasePictureInPicture(tabId, pictureInPictureSession, true); }, ); @@ -2061,118 +2068,154 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const closeAllPictureInPicture = Effect.fn("PreviewManager.closeAllPictureInPicture")( function* () { - const windows = yield* SynchronizedRef.get(pictureInPictureWindowsRef); - yield* Effect.forEach(windows.keys(), closePictureInPicture, { + const sessions = yield* SynchronizedRef.get(pictureInPictureSessionsRef); + yield* Effect.forEach(sessions.keys(), closePictureInPicture, { concurrency: "unbounded", discard: true, }); }, ); - const openPictureInPictureUnlocked = Effect.fn("PreviewManager.openPictureInPictureUnlocked")( - function* (tabId: string) { - const wc = yield* requireWebContents(tabId); - const existing = (yield* SynchronizedRef.get(pictureInPictureWindowsRef)).get(tabId); - if (existing && !existing.isDestroyed()) { - yield* attempt({ operation: "pictureInPicture.showExisting", tabId }, () => - existing.showInactive(), - ); - return; - } - const title = wc.getTitle().trim(); - const pictureInPictureWindow = yield* attempt( - { - operation: "pictureInPicture.create", - tabId, - webContentsId: wc.id, - }, - () => - new BrowserWindow({ - width: PICTURE_IN_PICTURE_INITIAL_WIDTH, - height: PICTURE_IN_PICTURE_INITIAL_HEIGHT, - minWidth: PICTURE_IN_PICTURE_MIN_WIDTH, - minHeight: PICTURE_IN_PICTURE_MIN_HEIGHT, - title: title.length > 0 ? `Preview · ${title}` : "Browser preview", - show: false, - alwaysOnTop: true, - autoHideMenuBar: true, - fullscreenable: false, - maximizable: false, - minimizable: false, - resizable: true, - skipTaskbar: true, - backgroundColor: "#111111", - ...(hostPlatform === "darwin" ? { type: "panel" as const } : {}), - webPreferences: { - preload: pictureInPicturePreloadPath, - backgroundThrottling: false, - contextIsolation: true, - nodeIntegration: false, - sandbox: true, - }, - }), - ); - const onClosed = () => { - runFork( - pictureInPictureMutationSemaphore.withPermit( - releasePictureInPicture(tabId, pictureInPictureWindow, false), - ), - ); - }; - yield* attempt( - { - operation: "pictureInPicture.configure", - tabId, - webContentsId: wc.id, - }, - () => { - pictureInPictureWindow.once("closed", onClosed); - pictureInPictureWindow.setAlwaysOnTop( - true, - hostPlatform === "darwin" ? "floating" : "normal", + const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( + tabId: string, + ) { + const wc = yield* requireWebContents(tabId); + const pictureInPictureSession = yield* pictureInPictureMutationSemaphore.withPermit( + Effect.gen(function* () { + const existing = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (existing && !existing.window.isDestroyed()) { + yield* attempt({ operation: "pictureInPicture.showExisting", tabId }, () => + existing.window.showInactive(), ); - if (hostPlatform === "darwin") { - pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { - visibleOnFullScreen: true, - }); - } - }, - ); - yield* SynchronizedRef.update(pictureInPictureWindowsRef, (windows) => - replaceMap(windows, (copy) => { - copy.set(tabId, pictureInPictureWindow); - }), - ); - const initialize = Effect.gen(function* () { - yield* attemptPromise( + return null; + } + if (existing) { + yield* releasePictureInPicture(tabId, existing, false); + } + const title = wc.getTitle().trim(); + const pictureInPictureWindow = yield* attempt( { - operation: "pictureInPicture.load", + operation: "pictureInPicture.create", tabId, webContentsId: wc.id, }, - () => pictureInPictureWindow.loadURL(buildPreviewPictureInPictureDataUrl()), + () => + new BrowserWindow({ + width: PICTURE_IN_PICTURE_INITIAL_WIDTH, + height: PICTURE_IN_PICTURE_INITIAL_HEIGHT, + minWidth: PICTURE_IN_PICTURE_MIN_WIDTH, + minHeight: PICTURE_IN_PICTURE_MIN_HEIGHT, + title: title.length > 0 ? `Preview · ${title}` : "Browser preview", + show: false, + alwaysOnTop: true, + autoHideMenuBar: true, + fullscreenable: false, + maximizable: false, + minimizable: false, + resizable: true, + skipTaskbar: true, + backgroundColor: "#111111", + ...(hostPlatform === "darwin" ? { type: "panel" as const } : {}), + webPreferences: { + preload: pictureInPicturePreloadPath, + backgroundThrottling: false, + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }), ); - yield* startFrameCapture(tabId, "picture-in-picture"); + const initializationScope = yield* Scope.fork(parentScope, "sequential"); + const session: PictureInPictureSession = { + window: pictureInPictureWindow, + initializationScope, + }; + const onClosed = () => { + runFork( + pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, session, false), + ), + ); + }; yield* attempt( { - operation: "pictureInPicture.show", + operation: "pictureInPicture.configure", tabId, webContentsId: wc.id, }, - () => pictureInPictureWindow.showInactive(), + () => { + pictureInPictureWindow.once("closed", onClosed); + pictureInPictureWindow.setAlwaysOnTop( + true, + hostPlatform === "darwin" ? "floating" : "normal", + ); + if (hostPlatform === "darwin") { + pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { + visibleOnFullScreen: true, + }); + } + }, + ).pipe( + Effect.onError(() => + Effect.all( + [ + Scope.close(initializationScope, Exit.void).pipe(Effect.ignore), + attempt({ operation: "pictureInPicture.close", tabId }, () => + pictureInPictureWindow.close(), + ).pipe(Effect.ignore), + ], + { discard: true }, + ), + ), ); - yield* update(tabId, { pictureInPicture: true }); - }); - yield* initialize.pipe( - Effect.onError(() => releasePictureInPicture(tabId, pictureInPictureWindow, true)), - ); - }, - ); + yield* SynchronizedRef.update(pictureInPictureSessionsRef, (sessions) => + replaceMap(sessions, (copy) => { + copy.set(tabId, session); + }), + ); + return session; + }), + ); + if (!pictureInPictureSession) return; - const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( - tabId: string, - ) { - yield* pictureInPictureMutationSemaphore.withPermit(openPictureInPictureUnlocked(tabId)); + const initialize = Effect.gen(function* () { + yield* attemptPromise( + { + operation: "pictureInPicture.load", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureSession.window.loadURL(buildPreviewPictureInPictureDataUrl()), + ); + yield* startFrameCapture(tabId, "picture-in-picture"); + yield* attempt( + { + operation: "pictureInPicture.show", + tabId, + webContentsId: wc.id, + }, + () => pictureInPictureSession.window.showInactive(), + ); + yield* update(tabId, { pictureInPicture: true }); + }); + const initializationFiber = yield* Effect.forkIn( + initialize, + pictureInPictureSession.initializationScope, + ); + const initializationExit = yield* Fiber.await(initializationFiber).pipe( + Effect.onInterrupt(() => + pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, pictureInPictureSession, true), + ), + ), + ); + if (Exit.isSuccess(initializationExit)) return; + const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (current !== pictureInPictureSession) return; + yield* pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, pictureInPictureSession, true), + ); + return yield* Effect.failCause(initializationExit.cause); }); const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { From 12d14b5d84e6910e4c0ede21e3537e7d1bd6113a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 01:18:33 +0200 Subject: [PATCH 05/26] fix(desktop): coordinate PiP session readiness Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 76 +++++++++++++++++++- apps/desktop/src/preview/Manager.ts | 91 ++++++++++++++++++------ 2 files changed, 144 insertions(+), 23 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index e20e9c7f944..8bf8e9da430 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -983,11 +983,15 @@ describe("PreviewManager", () => { expect(browserWindowConstructor).toHaveBeenCalledOnce(); expect(initializingWindow.loadURL).toHaveBeenCalledOnce(); expect(initializingWindow.close).toHaveBeenCalledOnce(); - yield* Fiber.join(firstOpen); - yield* Fiber.join(secondOpen); + const [firstOpenExit, secondOpenExit] = yield* Effect.all([ + Fiber.await(firstOpen), + Fiber.await(secondOpen), + ]); yield* Fiber.join(close); - expect(initializingWindow.showInactive).toHaveBeenCalledOnce(); + expect(Exit.hasInterrupts(firstOpenExit)).toBe(true); + expect(Exit.hasInterrupts(secondOpenExit)).toBe(true); + expect(initializingWindow.showInactive).not.toHaveBeenCalled(); expect(capturePage).not.toHaveBeenCalled(); expect(states.at(-1)?.pictureInPicture).toBe(false); @@ -1010,6 +1014,72 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("rejects picture-in-picture when its webview changes during initialization", () => + withManager((manager) => + Effect.gen(function* () { + const initialCapturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("stale-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const replacementCapturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("replacement-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const initialWebContents = makeTestPreviewWebContents(initialCapturePage, 42); + const replacementWebContents = makeTestPreviewWebContents(replacementCapturePage, 43); + fromId.mockImplementation((webContentsId?: number) => { + if (webContentsId === 42) return initialWebContents; + if (webContentsId === 43) return replacementWebContents; + return null; + }); + let resolveLoad: (() => void) | undefined; + const { pictureInPictureWindow } = makeTestPictureInPictureWindow( + () => + new Promise((resolve) => { + resolveLoad = resolve; + }), + ); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_replaced_webview"); + yield* manager.registerWebview("tab_replaced_webview", 42); + const open = yield* manager + .openPictureInPicture("tab_replaced_webview") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(pictureInPictureWindow.loadURL).toHaveBeenCalledOnce(); + expect(resolveLoad).toBeDefined(); + const concurrentOpen = yield* manager + .openPictureInPicture("tab_replaced_webview") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + + yield* manager.registerWebview("tab_replaced_webview", 43); + resolveLoad?.(); + + const openExits = yield* Effect.all([Fiber.await(open), Fiber.await(concurrentOpen)]); + for (const openExit of openExits) { + expect(Exit.isFailure(openExit)).toBe(true); + if (Exit.isSuccess(openExit)) continue; + const error = Option.getOrThrow(Cause.findErrorOption(openExit.cause)); + expect(error).toMatchObject({ + _tag: "PreviewOperationError", + operation: "pictureInPicture.validateWebContents", + tabId: "tab_replaced_webview", + webContentsId: 42, + }); + } + expect(browserWindowConstructor).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.showInactive).not.toHaveBeenCalled(); + expect(initialCapturePage).not.toHaveBeenCalled(); + expect(replacementCapturePage).not.toHaveBeenCalled(); + }), + ), + ); + effectIt.effect("keeps element picking active during subframe navigation", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 6d00534e515..77bd436c9f7 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -33,6 +33,7 @@ import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -336,6 +337,8 @@ interface FrameCaptureSession { interface PictureInPictureSession { readonly window: BrowserWindow; + readonly webContentsId: number; + readonly ready: Deferred.Deferred; readonly initializationScope: Scope.Closeable; } @@ -2029,6 +2032,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ] as const; }); if (!removed) return; + yield* Deferred.interrupt(expectedSession.ready); yield* Scope.close(expectedSession.initializationScope, Exit.void).pipe(Effect.ignore); yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => replaceMap(aspectRatios, (copy) => { @@ -2079,20 +2083,24 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( tabId: string, ) { - const wc = yield* requireWebContents(tabId); - const pictureInPictureSession = yield* pictureInPictureMutationSemaphore.withPermit( + const claim = yield* pictureInPictureMutationSemaphore.withPermit( Effect.gen(function* () { const existing = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); if (existing && !existing.window.isDestroyed()) { - yield* attempt({ operation: "pictureInPicture.showExisting", tabId }, () => - existing.window.showInactive(), - ); - return null; + return { kind: "existing" as const, session: existing }; } if (existing) { yield* releasePictureInPicture(tabId, existing, false); } - const title = wc.getTitle().trim(); + const wc = yield* requireWebContents(tabId); + const title = yield* attempt( + { + operation: "pictureInPicture.readTitle", + tabId, + webContentsId: wc.id, + }, + () => wc.getTitle().trim(), + ); const pictureInPictureWindow = yield* attempt( { operation: "pictureInPicture.create", @@ -2126,8 +2134,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }), ); const initializationScope = yield* Scope.fork(parentScope, "sequential"); + const ready = yield* Deferred.make(); const session: PictureInPictureSession = { window: pictureInPictureWindow, + webContentsId: wc.id, + ready, initializationScope, }; const onClosed = () => { @@ -2173,48 +2184,88 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function copy.set(tabId, session); }), ); - return session; + return { kind: "created" as const, session }; }), ); - if (!pictureInPictureSession) return; + const pictureInPictureSession = claim.session; + if (claim.kind === "existing") { + yield* Deferred.await(pictureInPictureSession.ready); + return yield* pictureInPictureMutationSemaphore.withPermit( + Effect.gen(function* () { + const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (current !== pictureInPictureSession || pictureInPictureSession.window.isDestroyed()) { + return yield* new PreviewOperationError({ + operation: "pictureInPicture.showExisting", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + cause: new Error("Picture-in-picture session closed before it became visible."), + }); + } + yield* attempt( + { + operation: "pictureInPicture.showExisting", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + }, + () => pictureInPictureSession.window.showInactive(), + ); + }), + ); + } const initialize = Effect.gen(function* () { yield* attemptPromise( { operation: "pictureInPicture.load", tabId, - webContentsId: wc.id, + webContentsId: pictureInPictureSession.webContentsId, }, () => pictureInPictureSession.window.loadURL(buildPreviewPictureInPictureDataUrl()), ); + const currentWebContents = yield* requireWebContents(tabId); + if ( + currentWebContents.id !== pictureInPictureSession.webContentsId || + currentWebContents.isDestroyed() + ) { + return yield* new PreviewOperationError({ + operation: "pictureInPicture.validateWebContents", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + cause: new Error("Preview webview changed while picture-in-picture was opening."), + }); + } yield* startFrameCapture(tabId, "picture-in-picture"); yield* attempt( { operation: "pictureInPicture.show", tabId, - webContentsId: wc.id, + webContentsId: pictureInPictureSession.webContentsId, }, () => pictureInPictureSession.window.showInactive(), ); yield* update(tabId, { pictureInPicture: true }); }); - const initializationFiber = yield* Effect.forkIn( - initialize, - pictureInPictureSession.initializationScope, - ); - const initializationExit = yield* Fiber.await(initializationFiber).pipe( + const initializationExit = yield* Effect.gen(function* () { + const initializationFiber = yield* Effect.forkIn( + initialize, + pictureInPictureSession.initializationScope, + ); + return yield* Fiber.await(initializationFiber); + }).pipe( Effect.onInterrupt(() => pictureInPictureMutationSemaphore.withPermit( releasePictureInPicture(tabId, pictureInPictureSession, true), ), ), ); + yield* Deferred.done(pictureInPictureSession.ready, initializationExit); if (Exit.isSuccess(initializationExit)) return; const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); - if (current !== pictureInPictureSession) return; - yield* pictureInPictureMutationSemaphore.withPermit( - releasePictureInPicture(tabId, pictureInPictureSession, true), - ); + if (current === pictureInPictureSession) { + yield* pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, pictureInPictureSession, true), + ); + } return yield* Effect.failCause(initializationExit.cause); }); From 235155d0db5dadda3a23e707fed982ec4b923158 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 01:27:06 +0200 Subject: [PATCH 06/26] fix(desktop): publish PiP readiness atomically Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 33 ++++++++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 18 ++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 8bf8e9da430..7eb7430dd4c 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -936,6 +936,39 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("does not publish picture-in-picture readiness after window teardown", () => + withManager((manager) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("closing-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + pictureInPictureWindow.showInactive.mockImplementationOnce(() => { + pictureInPictureWindow.close(); + }); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + + yield* manager.createTab("tab_pip_teardown"); + yield* manager.registerWebview("tab_pip_teardown", 42); + const openExit = yield* Effect.exit(manager.openPictureInPicture("tab_pip_teardown")); + + expect(Exit.hasInterrupts(openExit)).toBe(true); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + }), + ), + ); + effectIt.effect("closes an initializing picture-in-picture without blocking later opens", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 77bd436c9f7..ef1ae0d5dc2 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2258,8 +2258,24 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ), ); + if (Exit.isSuccess(initializationExit)) { + const published = yield* pictureInPictureMutationSemaphore.withPermit( + Effect.gen(function* () { + const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (current !== pictureInPictureSession || pictureInPictureSession.window.isDestroyed()) { + if (current === pictureInPictureSession) { + yield* releasePictureInPicture(tabId, pictureInPictureSession, false); + } + return false; + } + yield* Deferred.done(pictureInPictureSession.ready, initializationExit); + return true; + }), + ); + if (published) return; + return yield* Deferred.await(pictureInPictureSession.ready); + } yield* Deferred.done(pictureInPictureSession.ready, initializationExit); - if (Exit.isSuccess(initializationExit)) return; const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); if (current === pictureInPictureSession) { yield* pictureInPictureMutationSemaphore.withPermit( From 43f1bc6d0886534b01a7a6c22f7d1b2c33aabfb4 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 01:46:18 +0200 Subject: [PATCH 07/26] fix(preview): harden capture lifecycle Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 58 +++++++++++++ apps/desktop/src/preview/Manager.ts | 84 ++++++++++++------- apps/web/src/browser/browserRecording.test.ts | 56 ++++++++++++- apps/web/src/browser/browserRecording.ts | 24 +++++- 4 files changed, 188 insertions(+), 34 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 7eb7430dd4c..ace91554469 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -2,6 +2,7 @@ import { it as effectIt } from "@effect/vitest"; import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -509,6 +510,63 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("detaches a webview registered while tab close cleanup is in flight", () => + withManager((manager) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("close-race-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const firstWebContents = makeTestPreviewWebContents(capturePage, 42); + const replacementWebContents = makeTestPreviewWebContents(capturePage, 43); + const replacementListenerSpies = replacementWebContents as unknown as { + readonly off: ReturnType; + readonly ipc: { readonly off: ReturnType }; + }; + fromId.mockImplementation((id) => { + if (id === 42) return firstWebContents; + if (id === 43) return replacementWebContents; + return null; + }); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_close_register_race"); + yield* manager.registerWebview("tab_close_register_race", 42); + yield* manager.openPictureInPicture("tab_close_register_race"); + + const closeCleanupPaused = yield* Deferred.make(); + const continueCloseCleanup = yield* Deferred.make(); + yield* manager.subscribeStateChanges((_tabId, state) => + !state.pictureInPicture && state.webContentsId === 42 + ? Deferred.succeed(closeCleanupPaused, undefined).pipe( + Effect.andThen(Deferred.await(continueCloseCleanup)), + ) + : Effect.void, + ); + + const closeFiber = yield* manager + .closeTab("tab_close_register_race") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(closeCleanupPaused); + yield* manager.registerWebview("tab_close_register_race", 43); + yield* Deferred.succeed(continueCloseCleanup, undefined); + yield* Fiber.join(closeFiber); + + expect(replacementListenerSpies.off).toHaveBeenCalledWith( + "did-navigate", + expect.any(Function), + ); + expect(replacementListenerSpies.ipc.off).toHaveBeenCalledWith( + "preview:human-input", + expect.any(Function), + ); + }), + ), + ); + effectIt.effect("keeps a main-frame load failure visible until a retry starts", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index ef1ae0d5dc2..d1d48059ea7 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1349,8 +1349,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { - const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (!tab) return; + if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; yield* Effect.all( [ cancelPickElement(tabId), @@ -1362,15 +1361,27 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function discard: true, }, ); - if (tab.webContentsId != null) { + const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if (!current) return [Option.none(), tabs] as const; + return [ + Option.some(current), + replaceMap(tabs, (copy) => { + copy.delete(tabId); + }), + ] as const; + }); + if (Option.isNone(tab)) return; + const closedTab = tab.value; + if (closedTab.webContentsId != null) { yield* Effect.all( - [detachControlSession(tab.webContentsId), detachListeners(tab.webContentsId)], + [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], { concurrency: 2, discard: true }, ); } const updatedAt = yield* currentIso; const closed: PreviewTabState = { - ...tab, + ...closedTab, webContentsId: null, navStatus: { kind: "Idle" }, canGoBack: false, @@ -1381,11 +1392,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function controller: "none", updatedAt, }; - yield* SynchronizedRef.update(tabsRef, (tabs) => - replaceMap(tabs, (copy) => { - copy.delete(tabId); - }), - ); yield* emit(tabId, closed); }); @@ -1444,7 +1450,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wc.getZoomFactor(), ); yield* attachListeners(tabId, wc); - runFork(restoreControlSession(tabId, wc)); const registeredAt = yield* currentIso; const registration = yield* SynchronizedRef.modify(tabsRef, (tabs) => { const current = tabs.get(tabId); @@ -1475,9 +1480,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ] as const; }); if (Option.isNone(registration)) { + yield* Effect.all([detachControlSession(webContentsId), detachListeners(webContentsId)], { + concurrency: 2, + discard: true, + }); return yield* new PreviewTabNotFoundError({ tabId }); } const { state: registered, pendingUrl } = registration.value; + runFork(restoreControlSession(tabId, wc)); yield* emit(tabId, registered); yield* attempt({ operation: "registerWebview.sendTheme", tabId, webContentsId }, () => wc.send(ANNOTATION_THEME_CHANNEL, annotationTheme), @@ -1771,14 +1781,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // session attaches so a concurrent setColorScheme is not overwritten with // a stale snapshot. const restoreControlSession = (tabId: string, wc: Electron.WebContents) => - ensureControlSession(wc).pipe( - Effect.andThen(SynchronizedRef.get(tabsRef)), - Effect.flatMap((tabs) => { - const colorScheme = tabs.get(tabId)?.colorScheme ?? "system"; - return colorScheme === "system" ? Effect.void : applyColorScheme(tabId, wc, colorScheme); - }), - Effect.ignore, - ); + Effect.gen(function* () { + const beforeAttach = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (beforeAttach?.webContentsId !== wc.id) return; + yield* ensureControlSession(wc); + const afterAttach = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (afterAttach?.webContentsId !== wc.id) { + yield* detachControlSession(wc.id); + return; + } + if (afterAttach.colorScheme !== "system") { + yield* attemptPromise({ operation: "applyColorScheme", tabId, webContentsId: wc.id }, () => + wc.debugger.sendCommand("Emulation.setEmulatedMedia", { + features: [ + { + name: "prefers-color-scheme", + value: afterAttach.colorScheme, + }, + ], + }), + ); + } + }).pipe(Effect.ignore); const setColorScheme = Effect.fn("PreviewManager.setColorScheme")(function* ( tabId: string, @@ -1970,6 +1994,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, consumer: FrameCaptureConsumer, ) { + const captureNextFrame = Effect.sleep(RECORDING_FRAME_INTERVAL_MS).pipe( + Effect.andThen(capturePreviewFrame(tabId)), + Effect.catch((error) => + Effect.logWarning("Background preview frame capture failed.", { + tabId, + error, + }), + ), + ); const created = yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => { const current = sessions.get(tabId); if (current) { @@ -1988,6 +2021,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } return Effect.gen(function* () { const scope = yield* Scope.fork(parentScope, "sequential"); + yield* Effect.forkIn(Effect.forever(captureNextFrame), scope); return [ true, replaceMap(sessions, (copy) => { @@ -2000,18 +2034,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); }); if (!created) return; - const session = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); - if (!session) return; - const captureNextFrame = Effect.sleep(RECORDING_FRAME_INTERVAL_MS).pipe( - Effect.andThen(capturePreviewFrame(tabId)), - Effect.catch((error) => - Effect.logWarning("Background preview frame capture failed.", { - tabId, - error, - }), - ), - ); - yield* Effect.forkIn(Effect.forever(captureNextFrame), session.scope); yield* capturePreviewFrame(tabId).pipe(Effect.onError(() => stopFrameCapture(tabId, consumer))); }); diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 26fd055eb3e..a0fc23a15da 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -8,7 +8,17 @@ const { events, onFrame, registrySet, save, startScreencast, stopScreencast, sur }; return { events, - onFrame: vi.fn(() => vi.fn()), + onFrame: vi.fn( + ( + _listener: (frame: { + readonly tabId: string; + readonly data: string; + readonly width: number; + readonly height: number; + readonly receivedAt: string; + }) => void, + ) => vi.fn(), + ), registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { events.push( value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, @@ -134,6 +144,50 @@ describe("browser recording", () => { await stopBrowserRecording("recording-tab"); }); + it("resizes a hidden recording canvas to the captured frame dimensions", async () => { + const drawImage = vi.fn(); + const canvas = { + width: 0, + height: 0, + captureStream: () => ({}), + getContext: () => ({ drawImage }), + }; + class FakeImage { + private loadListener: EventListenerOrEventListenerObject | undefined; + + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { + if (type === "load") this.loadListener = listener; + } + + set src(_value: string) { + const event = new Event("load"); + if (typeof this.loadListener === "function") this.loadListener(event); + else this.loadListener?.handleEvent(event); + } + } + vi.stubGlobal("Image", FakeImage as unknown as typeof Image); + vi.stubGlobal("document", { + createElement: () => canvas, + }); + surfaceState.byTabId = {}; + + await startBrowserRecording("recording-tab"); + const frameListener = onFrame.mock.calls[0]?.[0]; + expect(frameListener).toBeDefined(); + frameListener?.({ + tabId: "recording-tab", + data: "captured-frame", + width: 390, + height: 844, + receivedAt: "2026-06-26T00:00:00.000Z", + }); + + expect(canvas).toMatchObject({ width: 390, height: 844 }); + expect(drawImage).toHaveBeenCalledWith(expect.any(FakeImage), 0, 0, 390, 844); + + await stopBrowserRecording("recording-tab"); + }); + it("records separate tabs concurrently", async () => { surfaceState.byTabId = { ...surfaceState.byTabId, diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 7d1748113fa..926e656c533 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -88,6 +88,7 @@ interface ActiveRecording { readonly mimeType: string; readonly startedAt: string; readonly startupSettled: Promise; + frameSequence: number; lifecycle: BrowserRecordingLifecycle; } @@ -116,12 +117,30 @@ const preferredMimeType = (): string => { const drawFrame = (frame: DesktopPreviewRecordingFrame): void => { const recording = activeRecordings.get(frame.tabId); if (!recording) return; + if ( + !Number.isFinite(frame.width) || + !Number.isFinite(frame.height) || + frame.width <= 0 || + frame.height <= 0 + ) { + return; + } + const width = Math.max(1, Math.round(frame.width)); + const height = Math.max(1, Math.round(frame.height)); + const frameSequence = ++recording.frameSequence; const image = new Image(); image.addEventListener( "load", () => { - if (activeRecordings.get(frame.tabId) !== recording) return; - recording.context.drawImage(image, 0, 0, recording.canvas.width, recording.canvas.height); + if ( + activeRecordings.get(frame.tabId) !== recording || + recording.frameSequence !== frameSequence + ) { + return; + } + if (recording.canvas.width !== width) recording.canvas.width = width; + if (recording.canvas.height !== height) recording.canvas.height = height; + recording.context.drawImage(image, 0, 0, width, height); }, { once: true }, ); @@ -246,6 +265,7 @@ export async function startBrowserRecording(tabId: string): Promise { mimeType, startedAt, startupSettled, + frameSequence: 0, lifecycle: { phase: "starting" }, }; activeRecordings.set(tabId, recording); From 65dbe947f0c7498892a531bc5d59ca909bfdf73c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 01:56:48 +0200 Subject: [PATCH 08/26] fix(recording): lock dimensions before encoding Co-authored-by: codex --- apps/web/src/browser/browserRecording.test.ts | 217 ++++++++++++---- apps/web/src/browser/browserRecording.ts | 242 ++++++++++++------ 2 files changed, 321 insertions(+), 138 deletions(-) diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index a0fc23a15da..869f54c848e 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -1,44 +1,72 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -const { events, onFrame, registrySet, save, startScreencast, stopScreencast, surfaceState } = - vi.hoisted(() => { - const events: string[] = []; - const surfaceState = { - byTabId: {} as Record, - }; - return { - events, - onFrame: vi.fn( - ( - _listener: (frame: { - readonly tabId: string; - readonly data: string; - readonly width: number; - readonly height: number; - readonly receivedAt: string; - }) => void, - ) => vi.fn(), - ), - registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { - events.push( - value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, - ); - }), - save: vi.fn(async (tabId: string) => ({ - id: "recording-test", +const { + events, + frameSubscription, + onFrame, + registrySet, + save, + startScreencast, + stopScreencast, + surfaceState, +} = vi.hoisted(() => { + const events: string[] = []; + type Frame = { + readonly tabId: string; + readonly data: string; + readonly width: number; + readonly height: number; + readonly receivedAt: string; + }; + const frameSubscription: { listener: ((frame: Frame) => void) | null } = { + listener: null, + }; + const surfaceState = { + byTabId: {} as Record, + }; + return { + events, + frameSubscription, + onFrame: vi.fn((listener: (frame: Frame) => void) => { + frameSubscription.listener = listener; + return () => { + if (frameSubscription.listener === listener) frameSubscription.listener = null; + }; + }), + registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { + events.push( + value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, + ); + }), + save: vi.fn(async (tabId: string) => ({ + id: "recording-test", + tabId, + path: "/tmp/recording-test.webm", + mimeType: "video/webm" as const, + sizeBytes: 0, + createdAt: "2026-06-26T00:00:00.000Z", + })), + startScreencast: vi.fn(async (tabId: string) => { + events.push("start-screencast"); + const surface = surfaceState.byTabId[tabId] as + | { + readonly content?: { readonly width: number; readonly height: number }; + readonly rect?: { readonly width: number; readonly height: number }; + } + | undefined; + const size = surface?.content ?? surface?.rect; + frameSubscription.listener?.({ tabId, - path: "/tmp/recording-test.webm", - mimeType: "video/webm" as const, - sizeBytes: 0, - createdAt: "2026-06-26T00:00:00.000Z", - })), - startScreencast: vi.fn(async () => { - events.push("start-screencast"); - }), - stopScreencast: vi.fn(async () => undefined), - surfaceState, - }; - }); + data: "initial-frame", + width: size?.width ?? 1280, + height: size?.height ?? 800, + receivedAt: "2026-06-26T00:00:00.000Z", + }); + }), + stopScreencast: vi.fn(async () => undefined), + surfaceState, + }; +}); vi.mock("~/components/preview/previewBridge", () => ({ previewBridge: { @@ -94,6 +122,7 @@ class FakeMediaRecorder { describe("browser recording", () => { beforeEach(() => { events.length = 0; + frameSubscription.listener = null; surfaceState.byTabId = { "recording-tab": { visible: true, @@ -104,12 +133,26 @@ describe("browser recording", () => { vi.clearAllMocks(); vi.stubGlobal("window", globalThis); vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder); + class ImmediateImage { + private loadListener: EventListenerOrEventListenerObject | undefined; + + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { + if (type === "load") this.loadListener = listener; + } + + set src(_value: string) { + const event = new Event("load"); + if (typeof this.loadListener === "function") this.loadListener(event); + else this.loadListener?.handleEvent(event); + } + } + vi.stubGlobal("Image", ImmediateImage as unknown as typeof Image); vi.stubGlobal("document", { createElement: () => ({ width: 0, height: 0, captureStream: () => ({}), - getContext: () => ({ drawImage: vi.fn() }), + getContext: () => ({ drawImage: vi.fn(), fillRect: vi.fn(), fillStyle: "" }), }), }); }); @@ -144,46 +187,108 @@ describe("browser recording", () => { await stopBrowserRecording("recording-tab"); }); - it("resizes a hidden recording canvas to the captured frame dimensions", async () => { + it("fixes hidden recording dimensions before MediaRecorder starts", async () => { const drawImage = vi.fn(); + const fillRect = vi.fn(); + let capturedStreamSize: { readonly width: number; readonly height: number } | undefined; const canvas = { width: 0, height: 0, - captureStream: () => ({}), - getContext: () => ({ drawImage }), + captureStream: () => { + capturedStreamSize = { width: canvas.width, height: canvas.height }; + return {}; + }, + getContext: () => ({ drawImage, fillRect, fillStyle: "" }), }; - class FakeImage { + vi.stubGlobal("document", { + createElement: () => canvas, + }); + surfaceState.byTabId = {}; + startScreencast.mockImplementationOnce(async (tabId: string) => { + events.push("start-screencast"); + frameSubscription.listener?.({ + tabId, + data: "captured-frame", + width: 390, + height: 844, + receivedAt: "2026-06-26T00:00:00.000Z", + }); + }); + + await startBrowserRecording("recording-tab"); + + expect(canvas).toMatchObject({ width: 390, height: 844 }); + expect(capturedStreamSize).toEqual({ width: 390, height: 844 }); + expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 390, 844); + + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "different-sized-frame", + width: 1280, + height: 720, + receivedAt: "2026-06-26T00:00:01.000Z", + }); + + expect(canvas).toMatchObject({ width: 390, height: 844 }); + expect(fillRect).toHaveBeenLastCalledWith(0, 0, 390, 844); + + await stopBrowserRecording("recording-tab"); + }); + + it("draws the newest decoded frames without starving behind decode latency", async () => { + const drawImage = vi.fn(); + class DeferredImage { + static readonly instances: DeferredImage[] = []; private loadListener: EventListenerOrEventListenerObject | undefined; + constructor() { + DeferredImage.instances.push(this); + } + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { if (type === "load") this.loadListener = listener; } - set src(_value: string) { + set src(_value: string) {} + + finishLoading(): void { const event = new Event("load"); if (typeof this.loadListener === "function") this.loadListener(event); else this.loadListener?.handleEvent(event); } } - vi.stubGlobal("Image", FakeImage as unknown as typeof Image); + vi.stubGlobal("Image", DeferredImage as unknown as typeof Image); vi.stubGlobal("document", { - createElement: () => canvas, + createElement: () => ({ + width: 0, + height: 0, + captureStream: () => ({}), + getContext: () => ({ drawImage, fillRect: vi.fn(), fillStyle: "" }), + }), }); - surfaceState.byTabId = {}; await startBrowserRecording("recording-tab"); - const frameListener = onFrame.mock.calls[0]?.[0]; - expect(frameListener).toBeDefined(); - frameListener?.({ + frameSubscription.listener?.({ tabId: "recording-tab", - data: "captured-frame", - width: 390, - height: 844, - receivedAt: "2026-06-26T00:00:00.000Z", + data: "second-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:01.000Z", + }); + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "third-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:02.000Z", }); - expect(canvas).toMatchObject({ width: 390, height: 844 }); - expect(drawImage).toHaveBeenCalledWith(expect.any(FakeImage), 0, 0, 390, 844); + DeferredImage.instances[1]?.finishLoading(); + expect(drawImage).toHaveBeenCalledOnce(); + DeferredImage.instances[2]?.finishLoading(); + expect(drawImage).toHaveBeenCalledTimes(2); + DeferredImage.instances[0]?.finishLoading(); + expect(drawImage).toHaveBeenCalledTimes(2); await stopBrowserRecording("recording-tab"); }); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 926e656c533..3b7399b0ed3 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -83,12 +83,16 @@ interface ActiveRecording { readonly tabId: string; readonly canvas: HTMLCanvasElement; readonly context: CanvasRenderingContext2D; - readonly recorder: MediaRecorder; readonly chunks: Blob[]; - readonly mimeType: string; readonly startedAt: string; readonly startupSettled: Promise; + readonly firstFrameSize: Promise; + readonly settleFirstFrameSize: () => void; + recorder: MediaRecorder | null; + mimeType: string | null; + frameSizeEstablished: boolean; frameSequence: number; + lastDrawnFrameSequence: number; lifecycle: BrowserRecordingLifecycle; } @@ -108,6 +112,7 @@ const activeRecordings = new Map(); let unsubscribeFrames: (() => void) | null = null; export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000; +const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 250; const preferredMimeType = (): string => { const candidates = ["video/mp4;codecs=avc1.42E01E", "video/webm;codecs=vp9", "video/webm"]; @@ -127,6 +132,12 @@ const drawFrame = (frame: DesktopPreviewRecordingFrame): void => { } const width = Math.max(1, Math.round(frame.width)); const height = Math.max(1, Math.round(frame.height)); + if (!recording.frameSizeEstablished) { + recording.canvas.width = width; + recording.canvas.height = height; + recording.frameSizeEstablished = true; + recording.settleFirstFrameSize(); + } const frameSequence = ++recording.frameSequence; const image = new Image(); image.addEventListener( @@ -134,21 +145,27 @@ const drawFrame = (frame: DesktopPreviewRecordingFrame): void => { () => { if ( activeRecordings.get(frame.tabId) !== recording || - recording.frameSequence !== frameSequence + frameSequence <= recording.lastDrawnFrameSequence ) { return; } - if (recording.canvas.width !== width) recording.canvas.width = width; - if (recording.canvas.height !== height) recording.canvas.height = height; - recording.context.drawImage(image, 0, 0, width, height); + recording.lastDrawnFrameSequence = frameSequence; + const scale = Math.min(recording.canvas.width / width, recording.canvas.height / height); + const targetWidth = width * scale; + const targetHeight = height * scale; + const targetX = (recording.canvas.width - targetWidth) / 2; + const targetY = (recording.canvas.height - targetHeight) / 2; + recording.context.fillStyle = "#000000"; + recording.context.fillRect(0, 0, recording.canvas.width, recording.canvas.height); + recording.context.drawImage(image, targetX, targetY, targetWidth, targetHeight); }, { once: true }, ); image.src = `data:image/jpeg;base64,${frame.data}`; }; -const stopMediaRecorder = async (recorder: MediaRecorder): Promise => { - if (recorder.state === "inactive") return; +const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise => { + if (!recorder || recorder.state === "inactive") return; const stopped = new Promise((resolve) => recorder.addEventListener("stop", () => resolve(), { once: true }), ); @@ -158,6 +175,7 @@ const stopMediaRecorder = async (recorder: MediaRecorder): Promise => { const clearActiveRecording = (recording: ActiveRecording): void => { if (activeRecordings.get(recording.tabId) !== recording) return; + recording.settleFirstFrameSize(); activeRecordings.delete(recording.tabId); if (activeRecordings.size === 0) { unsubscribeFrames?.(); @@ -168,6 +186,32 @@ const clearActiveRecording = (recording: ActiveRecording): void => { }); }; +const cleanupFailedRecordingStart = async ( + bridge: NonNullable, + recording: ActiveRecording, +): Promise => { + const errors: unknown[] = []; + try { + await bridge.recording.stopScreencast(recording.tabId); + } catch (error) { + errors.push(error); + } + try { + await stopMediaRecorder(recording.recorder); + } catch (error) { + errors.push(error); + } finally { + clearActiveRecording(recording); + } + if (errors.length === 0) return undefined; + if (errors.length === 1) return errors[0]; + return new AggregateError( + errors, + `Browser recording startup cleanup failed for tab ${recording.tabId}.`, + { cause: errors[0] }, + ); +}; + const recordingStartupCancelledError = ( recording: ActiveRecording, cause: unknown = new Error(`Browser recording startup was cancelled for tab ${recording.tabId}.`), @@ -181,6 +225,19 @@ const recordingStartupCancelledError = ( const isRecordingStarting = (recording: ActiveRecording): boolean => activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting"; +const waitForFirstFrameSize = async (recording: ActiveRecording): Promise => { + if (recording.frameSizeEstablished) return; + let timeout: ReturnType | null = null; + await Promise.race([ + recording.firstFrameSize, + new Promise((resolve) => { + timeout = setTimeout(resolve, BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); + }), + ]); + if (timeout !== null) clearTimeout(timeout); + recording.frameSizeEstablished = true; +}; + const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise => { let timeout: ReturnType | null = null; try { @@ -232,40 +289,30 @@ export async function startBrowserRecording(tabId: string): Promise { height: canvas.height, }); } - let mimeType: string; - let recorder: MediaRecorder; - try { - mimeType = preferredMimeType(); - recorder = new MediaRecorder(canvas.captureStream(12), { - mimeType, - videoBitsPerSecond: 4_000_000, - }); - } catch (cause) { - throw new BrowserRecordingOperationError({ - operation: "initialize-media-recorder", - tabId, - cause, - }); - } const startedAt = new Date().toISOString(); const chunks: Blob[] = []; let settleStartup: (() => void) | undefined; const startupSettled = new Promise((resolve) => { settleStartup = resolve; }); - recorder.addEventListener("dataavailable", (event) => { - if (event.data.size > 0) chunks.push(event.data); + let settleFirstFrameSize: (() => void) | undefined; + const firstFrameSize = new Promise((resolve) => { + settleFirstFrameSize = resolve; }); const recording: ActiveRecording = { tabId, canvas, context, - recorder, chunks, - mimeType, startedAt, startupSettled, + firstFrameSize, + settleFirstFrameSize: () => settleFirstFrameSize?.(), + recorder: null, + mimeType: null, + frameSizeEstablished: false, frameSequence: 0, + lastDrawnFrameSequence: 0, lifecycle: { phase: "starting" }, }; activeRecordings.set(tabId, recording); @@ -280,47 +327,21 @@ export async function startBrowserRecording(tabId: string): Promise { cause, }); } - try { - recorder.start(1_000); - } catch (cause) { - clearActiveRecording(recording); - throw new BrowserRecordingOperationError({ - operation: "start-media-recorder", - tabId, - cause, - }); - } - if (!isRecordingStarting(recording)) { - throw recordingStartupCancelledError(recording); - } try { await bridge.recording.startScreencast(tabId); } catch (cause) { if (!isRecordingStarting(recording)) { throw recordingStartupCancelledError(recording, cause); } - let cleanupCause: unknown; - try { - await stopMediaRecorder(recorder); - } catch (error) { - cleanupCause = error; - } finally { - clearActiveRecording(recording); - } + clearActiveRecording(recording); throw new BrowserRecordingOperationError({ operation: "start-screencast", tabId, - cause: - cleanupCause === undefined - ? cause - : new AggregateError( - [cause, cleanupCause], - `Browser recording start and cleanup failed for tab ${tabId}.`, - { cause }, - ), + cause, }); } - if (!isRecordingStarting(recording)) { + const throwIfStartupCancelled = async (): Promise => { + if (isRecordingStarting(recording)) return; try { await bridge.recording.stopScreencast(tabId); } catch (cause) { @@ -334,6 +355,55 @@ export async function startBrowserRecording(tabId: string): Promise { ); } throw recordingStartupCancelledError(recording); + }; + await throwIfStartupCancelled(); + await waitForFirstFrameSize(recording); + await throwIfStartupCancelled(); + + let mimeType: string; + let recorder: MediaRecorder; + try { + mimeType = preferredMimeType(); + recorder = new MediaRecorder(canvas.captureStream(12), { + mimeType, + videoBitsPerSecond: 4_000_000, + }); + recording.mimeType = mimeType; + recording.recorder = recorder; + recorder.addEventListener("dataavailable", (event) => { + if (event.data.size > 0) chunks.push(event.data); + }); + } catch (cause) { + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + throw new BrowserRecordingOperationError({ + operation: "initialize-media-recorder", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser recording initialization and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); + } + try { + recorder.start(1_000); + } catch (cause) { + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + throw new BrowserRecordingOperationError({ + operation: "start-media-recorder", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser media recorder start and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); } recording.lifecycle = { phase: "recording" }; appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { @@ -348,10 +418,13 @@ export async function startBrowserRecording(tabId: string): Promise { const finalizeBrowserRecording = async ( bridge: NonNullable, recording: ActiveRecording, -): Promise => { +): Promise => { const { tabId } = recording; let result: - | { readonly _tag: "Success"; readonly artifact: DesktopPreviewRecordingArtifact } + | { + readonly _tag: "Success"; + readonly artifact: DesktopPreviewRecordingArtifact | null; + } | { readonly _tag: "Failure"; readonly error: unknown }; try { try { @@ -364,29 +437,33 @@ const finalizeBrowserRecording = async ( }); } await waitForRecordingStartupToSettle(recording); - try { - await stopMediaRecorder(recording.recorder); - } catch (cause) { - throw new BrowserRecordingOperationError({ - operation: "stop-media-recorder", - tabId, - cause, - }); - } - try { - const blob = new Blob(recording.chunks, { type: recording.mimeType }); - const artifact = await bridge.recording.save( - tabId, - recording.mimeType, - new Uint8Array(await blob.arrayBuffer()), - ); - result = { _tag: "Success", artifact }; - } catch (cause) { - throw new BrowserRecordingOperationError({ - operation: "save-artifact", - tabId, - cause, - }); + if (!recording.recorder || !recording.mimeType) { + result = { _tag: "Success", artifact: null }; + } else { + try { + await stopMediaRecorder(recording.recorder); + } catch (cause) { + throw new BrowserRecordingOperationError({ + operation: "stop-media-recorder", + tabId, + cause, + }); + } + try { + const blob = new Blob(recording.chunks, { type: recording.mimeType }); + const artifact = await bridge.recording.save( + tabId, + recording.mimeType, + new Uint8Array(await blob.arrayBuffer()), + ); + result = { _tag: "Success", artifact }; + } catch (cause) { + throw new BrowserRecordingOperationError({ + operation: "save-artifact", + tabId, + cause, + }); + } } } catch (error) { result = { _tag: "Failure", error }; @@ -452,6 +529,7 @@ export function stopBrowserRecording( if (!bridge || !recording) return Promise.resolve(null); if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; + recording.settleFirstFrameSize(); const stopPromise = Promise.resolve() .then(() => finalizeBrowserRecording(bridge, recording)) .catch((error) => { From 7b046af10868e60c2ec34cdd8b57710bcb832ad5 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 02:04:59 +0200 Subject: [PATCH 09/26] fix(recording): require stable startup frames Co-authored-by: codex --- apps/web/src/browser/browserRecording.test.ts | 61 +++++++++++++++- apps/web/src/browser/browserRecording.ts | 70 ++++++++++++++----- 2 files changed, 112 insertions(+), 19 deletions(-) diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 869f54c848e..4fd2cde5db7 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -85,6 +85,7 @@ vi.mock("./browserSurfaceStore", () => ({ })); import { + BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS, BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS, BrowserRecordingConflictError, BrowserRecordingOperationError, @@ -187,6 +188,25 @@ describe("browser recording", () => { await stopBrowserRecording("recording-tab"); }); + it("fails startup instead of locking a fallback size when no frame arrives", async () => { + vi.useFakeTimers(); + startScreencast.mockImplementationOnce(async () => { + events.push("start-screencast"); + }); + + const startPromise = startBrowserRecording("recording-tab"); + const rejection = expect(startPromise).rejects.toMatchObject({ + operation: "wait-first-frame", + tabId: "recording-tab", + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); + + await rejection; + expect(stopScreencast).toHaveBeenCalledWith("recording-tab"); + expect(events.at(-1)).toBe("clear"); + }); + it("fixes hidden recording dimensions before MediaRecorder starts", async () => { const drawImage = vi.fn(); const fillRect = vi.fn(); @@ -321,8 +341,15 @@ describe("browser recording", () => { it("does not report success for a second start while the first is still starting", async () => { let finishStartingScreencast: (() => void) | undefined; - startScreencast.mockImplementationOnce(async () => { + startScreencast.mockImplementationOnce(async (tabId: string) => { events.push("start-screencast"); + frameSubscription.listener?.({ + tabId, + data: "initial-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:00.000Z", + }); await new Promise((resolve) => { finishStartingScreencast = resolve; }); @@ -437,6 +464,38 @@ describe("browser recording", () => { expect(startCallsBeforeFirstSettled).toBe(1); }); + it("keeps the recording slot while a failed stop waits for startup", async () => { + let finishStartingScreencast: (() => void) | undefined; + startScreencast.mockImplementationOnce(async () => { + events.push("start-screencast"); + await new Promise((resolve) => { + finishStartingScreencast = resolve; + }); + }); + stopScreencast.mockRejectedValueOnce(new Error("initial stop failed")); + + const firstStart = startBrowserRecording("recording-tab"); + const rejectedStart = expect(firstStart).rejects.toBeInstanceOf(BrowserRecordingOperationError); + await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); + + const stopPromise = stopBrowserRecording("recording-tab"); + const rejectedStop = expect(stopPromise).rejects.toMatchObject({ + operation: "stop-screencast", + tabId: "recording-tab", + }); + await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce()); + await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf( + BrowserRecordingConflictError, + ); + + finishStartingScreencast?.(); + await rejectedStart; + await rejectedStop; + + await startBrowserRecording("recording-tab"); + await stopBrowserRecording("recording-tab"); + }); + it("fails a stop that waits too long for startup without freeing the recording slot", async () => { vi.useFakeTimers(); let finishStartingScreencast: (() => void) | undefined; diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 3b7399b0ed3..63c67800687 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -55,6 +55,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass; - readonly firstFrameSize: Promise; - readonly settleFirstFrameSize: () => void; + readonly firstFrameSize: Promise<"frame" | "cancelled">; + readonly settleFirstFrameSize: (outcome: "frame" | "cancelled") => void; recorder: MediaRecorder | null; mimeType: string | null; frameSizeEstablished: boolean; @@ -112,7 +113,7 @@ const activeRecordings = new Map(); let unsubscribeFrames: (() => void) | null = null; export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000; -const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 250; +export const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 5_000; const preferredMimeType = (): string => { const candidates = ["video/mp4;codecs=avc1.42E01E", "video/webm;codecs=vp9", "video/webm"]; @@ -136,7 +137,7 @@ const drawFrame = (frame: DesktopPreviewRecordingFrame): void => { recording.canvas.width = width; recording.canvas.height = height; recording.frameSizeEstablished = true; - recording.settleFirstFrameSize(); + recording.settleFirstFrameSize("frame"); } const frameSequence = ++recording.frameSequence; const image = new Image(); @@ -175,7 +176,7 @@ const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise const clearActiveRecording = (recording: ActiveRecording): void => { if (activeRecordings.get(recording.tabId) !== recording) return; - recording.settleFirstFrameSize(); + recording.settleFirstFrameSize("cancelled"); activeRecordings.delete(recording.tabId); if (activeRecordings.size === 0) { unsubscribeFrames?.(); @@ -225,17 +226,17 @@ const recordingStartupCancelledError = ( const isRecordingStarting = (recording: ActiveRecording): boolean => activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting"; -const waitForFirstFrameSize = async (recording: ActiveRecording): Promise => { - if (recording.frameSizeEstablished) return; +const waitForFirstFrameSize = async (recording: ActiveRecording): Promise => { + if (recording.frameSizeEstablished) return true; let timeout: ReturnType | null = null; - await Promise.race([ + const outcome = await Promise.race([ recording.firstFrameSize, - new Promise((resolve) => { - timeout = setTimeout(resolve, BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); + new Promise<"timeout">((resolve) => { + timeout = setTimeout(() => resolve("timeout"), BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); }), ]); if (timeout !== null) clearTimeout(timeout); - recording.frameSizeEstablished = true; + return outcome === "frame"; }; const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise => { @@ -295,8 +296,8 @@ export async function startBrowserRecording(tabId: string): Promise { const startupSettled = new Promise((resolve) => { settleStartup = resolve; }); - let settleFirstFrameSize: (() => void) | undefined; - const firstFrameSize = new Promise((resolve) => { + let settleFirstFrameSize: ((outcome: "frame" | "cancelled") => void) | undefined; + const firstFrameSize = new Promise<"frame" | "cancelled">((resolve) => { settleFirstFrameSize = resolve; }); const recording: ActiveRecording = { @@ -307,7 +308,7 @@ export async function startBrowserRecording(tabId: string): Promise { startedAt, startupSettled, firstFrameSize, - settleFirstFrameSize: () => settleFirstFrameSize?.(), + settleFirstFrameSize: (outcome) => settleFirstFrameSize?.(outcome), recorder: null, mimeType: null, frameSizeEstablished: false, @@ -357,8 +358,24 @@ export async function startBrowserRecording(tabId: string): Promise { throw recordingStartupCancelledError(recording); }; await throwIfStartupCancelled(); - await waitForFirstFrameSize(recording); + const hasFirstFrame = await waitForFirstFrameSize(recording); await throwIfStartupCancelled(); + if (!hasFirstFrame) { + const cause = new Error(`No valid recording frame arrived for tab ${tabId}.`); + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + throw new BrowserRecordingOperationError({ + operation: "wait-first-frame", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser recording frame wait and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); + } let mimeType: string; let recorder: MediaRecorder; @@ -427,16 +444,33 @@ const finalizeBrowserRecording = async ( } | { readonly _tag: "Failure"; readonly error: unknown }; try { + let stopScreencastError: BrowserRecordingOperationError | undefined; try { await bridge.recording.stopScreencast(tabId); } catch (cause) { - throw new BrowserRecordingOperationError({ + stopScreencastError = new BrowserRecordingOperationError({ operation: "stop-screencast", tabId, cause, }); } - await waitForRecordingStartupToSettle(recording); + try { + await waitForRecordingStartupToSettle(recording); + } catch (startupError) { + if (stopScreencastError) { + throw new BrowserRecordingOperationError({ + operation: "wait-startup", + tabId, + cause: new AggregateError( + [startupError, stopScreencastError], + `Browser recording stop failed while startup remained pending for tab ${tabId}.`, + { cause: startupError }, + ), + }); + } + throw startupError; + } + if (stopScreencastError) throw stopScreencastError; if (!recording.recorder || !recording.mimeType) { result = { _tag: "Success", artifact: null }; } else { @@ -529,7 +563,7 @@ export function stopBrowserRecording( if (!bridge || !recording) return Promise.resolve(null); if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; - recording.settleFirstFrameSize(); + recording.settleFirstFrameSize("cancelled"); const stopPromise = Promise.resolve() .then(() => finalizeBrowserRecording(bridge, recording)) .catch((error) => { From 117d0974e790a00acf62139b0463f89dc154f9de Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 12:32:46 +0200 Subject: [PATCH 10/26] feat(preview): add thread-scoped floating browser Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 42 +++ apps/desktop/src/preview/Manager.ts | 15 +- apps/server/src/preview/Manager.test.ts | 26 ++ apps/server/src/preview/Manager.ts | 85 +++--- apps/server/src/server.test.ts | 2 +- apps/web/src/browser/BrowserSurfaceSlot.tsx | 19 +- apps/web/src/browser/HostedBrowserWebview.tsx | 15 +- .../src/browser/browserSurfaceStore.test.ts | 18 +- apps/web/src/browser/browserSurfaceStore.ts | 23 +- .../browser/hostedBrowserWebviewStyle.test.ts | 17 ++ .../src/browser/hostedBrowserWebviewStyle.ts | 5 +- apps/web/src/components/ChatView.tsx | 35 +++ .../preview/PreviewAutomationHosts.tsx | 2 +- .../components/preview/PreviewChromeRow.tsx | 4 +- .../components/preview/PreviewMoreMenu.tsx | 11 + .../components/preview/PreviewView.test.tsx | 83 +++++- .../src/components/preview/PreviewView.tsx | 21 +- .../preview/ThreadPreviewMiniPlayer.tsx | 241 ++++++++++++++++++ .../preview/previewMiniPlayerLayout.test.ts | 34 +++ .../preview/previewMiniPlayerLayout.ts | 27 ++ .../components/preview/usePreviewSession.ts | 47 +--- apps/web/src/previewMiniPlayerStore.test.ts | 52 ++++ apps/web/src/previewMiniPlayerStore.ts | 75 ++++++ apps/web/src/previewStateStore.test.ts | 99 ++++++- apps/web/src/previewStateStore.ts | 124 +++++---- packages/contracts/src/preview.test.ts | 8 + packages/contracts/src/preview.ts | 10 +- 27 files changed, 994 insertions(+), 146 deletions(-) create mode 100644 apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx create mode 100644 apps/web/src/components/preview/previewMiniPlayerLayout.test.ts create mode 100644 apps/web/src/components/preview/previewMiniPlayerLayout.ts create mode 100644 apps/web/src/previewMiniPlayerStore.test.ts create mode 100644 apps/web/src/previewMiniPlayerStore.ts diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index ace91554469..9fbc632f052 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -959,6 +959,48 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("retries a cold hidden-tab capture without dropping recording", () => + withManager((manager) => + Effect.gen(function* () { + const jpeg = Buffer.from("recovered-preview-frame"); + const capturePage = vi.fn(async () => ({ + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + })); + capturePage.mockRejectedValueOnce(new Error("UnknownVizError")); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_cold_capture"); + yield* manager.registerWebview("tab_cold_capture", 42); + + yield* manager.startRecording("tab_cold_capture"); + + expect(capturePage).toHaveBeenCalledOnce(); + expect(frames).toHaveLength(0); + + yield* TestClock.adjust(100); + + expect(capturePage).toHaveBeenCalledTimes(2); + expect(frames).toEqual([ + expect.objectContaining({ + tabId: "tab_cold_capture", + data: jpeg.toString("base64"), + width: 1280, + height: 720, + }), + ]); + + yield* manager.stopRecording("tab_cold_capture"); + }), + ), + ); + effectIt.effect("drops empty frames before picture-in-picture delivery", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index d1d48059ea7..12a26c12017 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1994,6 +1994,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, consumer: FrameCaptureConsumer, ) { + // Validate the tab synchronously, but treat capturePage failures as + // transient. Chromium can return UnknownVizError while a hidden guest is + // warming its first compositor frame; the scheduled loop should keep the + // consumer alive and recover instead of tearing recording/PiP back down. + yield* requireWebContents(tabId); const captureNextFrame = Effect.sleep(RECORDING_FRAME_INTERVAL_MS).pipe( Effect.andThen(capturePreviewFrame(tabId)), Effect.catch((error) => @@ -2034,7 +2039,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); }); if (!created) return; - yield* capturePreviewFrame(tabId).pipe(Effect.onError(() => stopFrameCapture(tabId, consumer))); + yield* capturePreviewFrame(tabId).pipe( + Effect.catch((error) => + Effect.logWarning("Initial background preview frame was not ready; capture will retry.", { + tabId, + consumer, + error, + }), + ), + ); }); const releasePictureInPicture = Effect.fn("PreviewManager.releasePictureInPicture")(function* ( diff --git a/apps/server/src/preview/Manager.test.ts b/apps/server/src/preview/Manager.test.ts index 693111b578f..ff662cdaae4 100644 --- a/apps/server/src/preview/Manager.test.ts +++ b/apps/server/src/preview/Manager.test.ts @@ -67,6 +67,32 @@ it.layer(PreviewManager.layer)("PreviewManager", (it) => { }), ); + it.effect("orders list snapshots and events with one monotonic revision", () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + const collector = yield* collectEvents; + const before = yield* manager.list({ threadId }); + + const opened = yield* manager.open({ threadId, url: "http://localhost:5173" }); + yield* manager.navigate({ + threadId, + tabId: opened.tabId, + url: "http://localhost:5173/ready", + }); + + const events = yield* collector.drain; + const listed = yield* manager.list({ threadId }); + expect(events).toHaveLength(2); + expect(events[0]!.serverEpoch).toBe(listed.serverEpoch); + expect(events[1]!.serverEpoch).toBe(listed.serverEpoch); + expect(events[0]!.revision).toBeGreaterThan(before.revision); + expect(events[1]!.revision).toBeGreaterThan(events[0]!.revision); + expect(listed.revision).toBe(events[1]!.revision); + expect(listed.sessions).toHaveLength(1); + }), + ); + it.effect("treats bare hosts as https", () => Effect.gen(function* () { const threadId = freshThreadId(); diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index 193ea85b21b..c3487f6321b 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -30,6 +30,7 @@ import { newPreviewTabId, normalizePreviewUrl, } from "@t3tools/shared/preview"; +import * as NodeCrypto from "node:crypto"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -67,9 +68,17 @@ interface PreviewSessionState { interface ManagerState { /** All sessions across every thread, keyed by `${threadId}\u0000${tabId}`. */ readonly sessions: ReadonlyMap; + /** Global monotonic revision establishing list/event ordering. */ + readonly revision: number; } -const initialState: ManagerState = { sessions: new Map() }; +const initialState: ManagerState = { sessions: new Map(), revision: 0 }; + +type PreviewEventDraft = PreviewEvent extends infer Event + ? Event extends { readonly revision: number } + ? Omit + : never + : never; const compositeKey = (threadId: string, tabId: string): string => `${threadId}\u0000${tabId}`; @@ -138,6 +147,7 @@ const buildIdleSnapshot = (input: { }); export const make = Effect.gen(function* PreviewManagerMake() { + const serverEpoch = NodeCrypto.randomUUID(); const stateRef = yield* SynchronizedRef.make(initialState); // Unbounded PubSub is fine here — events are tiny and we don't want to // block publishers if a subscriber is slow. WS clients backpressure on @@ -160,7 +170,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { tabId: string, mutator: ( session: PreviewSessionState, - ) => Effect.Effect<{ next: PreviewSessionState; emit: PreviewEvent | null; result: R }, E>, + ) => Effect.Effect<{ next: PreviewSessionState; emit: PreviewEventDraft | null; result: R }, E>, ): Effect.Effect => { type ModifyResult = | { kind: "fail"; error: PreviewSessionLookupError } @@ -177,10 +187,17 @@ export const make = Effect.gen(function* PreviewManagerMake() { return mutator(session).pipe( Effect.flatMap( Effect.fn("PreviewManager.commitMutation")(function* ({ next, emit, result }) { - if (emit) yield* PubSub.publish(eventsPubSub, emit); + const revision = emit ? state.revision + 1 : state.revision; + if (emit) { + yield* PubSub.publish(eventsPubSub, { + ...emit, + revision, + serverEpoch, + } as PreviewEvent); + } const sessions = new Map(state.sessions); sessions.set(compositeKey(threadId, tabId), next); - return [{ kind: "ok", result } as ModifyResult, { sessions }] as readonly [ + return [{ kind: "ok", result } as ModifyResult, { sessions, revision }] as readonly [ ModifyResult, ManagerState, ]; @@ -207,22 +224,27 @@ export const make = Effect.gen(function* PreviewManagerMake() { updatedAt, }) : buildIdleSnapshot({ threadId: input.threadId, tabId, updatedAt }); - yield* SynchronizedRef.update(stateRef, (state) => { - const sessions = new Map(state.sessions); - sessions.set(compositeKey(input.threadId, tabId), { - threadId: input.threadId, - tabId, - snapshot, - }); - return { sessions }; - }); - yield* PubSub.publish(eventsPubSub, { - type: "opened", - threadId: input.threadId, - tabId, - createdAt: snapshot.updatedAt, - snapshot, - }); + yield* SynchronizedRef.modifyEffect(stateRef, (state) => + Effect.gen(function* () { + const revision = state.revision + 1; + const sessions = new Map(state.sessions); + sessions.set(compositeKey(input.threadId, tabId), { + threadId: input.threadId, + tabId, + snapshot, + }); + yield* PubSub.publish(eventsPubSub, { + type: "opened", + threadId: input.threadId, + tabId, + createdAt: snapshot.updatedAt, + serverEpoch, + revision, + snapshot, + }); + return [snapshot, { sessions, revision }] as const; + }), + ); return snapshot; }, ); @@ -280,7 +302,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { viewport: session.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, updatedAt, }; - const emit: PreviewEvent = + const emit: PreviewEventDraft = input.navStatus._tag === "LoadFailed" ? { type: "failed", @@ -349,7 +371,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { const close: PreviewManager["Service"]["close"] = Effect.fn("PreviewManager.close")( function* (input) { const createdAt = yield* currentIsoTimestamp; - const events = yield* SynchronizedRef.modify(stateRef, (state) => { + yield* SynchronizedRef.modifyEffect(stateRef, (state) => { const eventsToEmit: PreviewEvent[] = []; const sessions = new Map(state.sessions); const targets = input.tabId @@ -357,6 +379,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { (entry): entry is PreviewSessionState => entry !== undefined, ) : sessionsForThread(state, input.threadId); + const revision = targets.length > 0 ? state.revision + 1 : state.revision; for (const target of targets) { sessions.delete(compositeKey(target.threadId, target.tabId)); eventsToEmit.push({ @@ -364,18 +387,20 @@ export const make = Effect.gen(function* PreviewManagerMake() { threadId: target.threadId, tabId: target.tabId, createdAt, + serverEpoch, + revision, }); } if (eventsToEmit.length === 0) { - return [eventsToEmit, state] as const; + return Effect.succeed([undefined, state] as const); } - return [eventsToEmit, { sessions }] as const; + return Effect.as( + Effect.forEach(eventsToEmit, (event) => PubSub.publish(eventsPubSub, event), { + discard: true, + }), + [undefined, { sessions, revision }] as const, + ); }); - if (events.length > 0) { - yield* Effect.forEach(events, (event) => PubSub.publish(eventsPubSub, event), { - discard: true, - }); - } }, ); @@ -387,6 +412,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { sessions: sessionsForThread(state, input.threadId) .map((s) => s.snapshot) .toSorted((a, b) => a.updatedAt.localeCompare(b.updatedAt)), + serverEpoch, + revision: state.revision, }), ), ); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index 871b79eca90..1c9e1724702 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -666,7 +666,7 @@ const buildAppUnderTest = (options?: { reportStatus: () => Effect.void, refresh: () => Effect.void, close: () => Effect.void, - list: () => Effect.succeed({ sessions: [] }), + list: () => Effect.succeed({ sessions: [], serverEpoch: "test-server", revision: 0 }), events: Stream.empty, subscribeEvents: Effect.flatMap(PubSub.unbounded(), (pubsub) => PubSub.subscribe(pubsub), diff --git a/apps/web/src/browser/BrowserSurfaceSlot.tsx b/apps/web/src/browser/BrowserSurfaceSlot.tsx index de74cfa2a90..02b267dbec4 100644 --- a/apps/web/src/browser/BrowserSurfaceSlot.tsx +++ b/apps/web/src/browser/BrowserSurfaceSlot.tsx @@ -7,10 +7,14 @@ import { acquireBrowserSurface } from "./browserSurfaceStore"; export function BrowserSurfaceSlot(props: { readonly tabId: string; readonly visible: boolean; + readonly cornerRadius?: number; + readonly layoutVersion?: string | number; readonly className?: string; }) { - const { tabId, visible, className } = props; + const { tabId, visible, cornerRadius = 0, layoutVersion, className } = props; const elementRef = useRef(null); + const presentationRef = useRef({ visible, cornerRadius }); + const updateRef = useRef<(() => void) | null>(null); useLayoutEffect(() => { const element = elementRef.current; @@ -18,6 +22,7 @@ export function BrowserSurfaceSlot(props: { const lease = acquireBrowserSurface(tabId); const update = () => { const rect = element.getBoundingClientRect(); + const presentation = presentationRef.current; lease.present( { x: Math.round(rect.x), @@ -25,9 +30,11 @@ export function BrowserSurfaceSlot(props: { width: Math.max(1, Math.round(rect.width)), height: Math.max(1, Math.round(rect.height)), }, - visible && rect.width > 0 && rect.height > 0, + presentation.visible && rect.width > 0 && rect.height > 0, + presentation.cornerRadius, ); }; + updateRef.current = update; update(); const observer = new ResizeObserver(update); observer.observe(element); @@ -37,9 +44,15 @@ export function BrowserSurfaceSlot(props: { observer.disconnect(); window.removeEventListener("resize", update); window.removeEventListener("scroll", update, true); + if (updateRef.current === update) updateRef.current = null; lease.release(); }; - }, [tabId, visible]); + }, [tabId]); + + useLayoutEffect(() => { + presentationRef.current = { visible, cornerRadius }; + updateRef.current?.(); + }, [cornerRadius, layoutVersion, visible]); return
; } diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 4e38f570c35..e9bc1cb80fc 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -50,6 +50,7 @@ export function HostedBrowserWebview(props: { useShallow((state) => { const current = state.byTabId[tabId]; return { + cornerRadius: current?.cornerRadius ?? 0, rect: resolveBrowserSurfacePanelRect(state.byTabId, tabId), visible: current?.visible ?? false, }; @@ -66,6 +67,8 @@ export function HostedBrowserWebview(props: { }; }, [tabId]); + const [webviewGeneration, setWebviewGeneration] = useState(0); + const setWebviewRef = useCallback((node: HTMLElement | null) => { webviewRef.current = node as ElectronWebview | null; if (node && !node.hasAttribute("allowpopups")) node.setAttribute("allowpopups", "true"); @@ -95,15 +98,21 @@ export function HostedBrowserWebview(props: { } })(); }; + const recoverGuest = () => { + if (disposed) return; + setWebviewGeneration((generation) => generation + 1); + }; webview.addEventListener("did-attach", register); webview.addEventListener("dom-ready", register); + webview.addEventListener("render-process-gone", recoverGuest); register(); return () => { disposed = true; webview.removeEventListener("did-attach", register); webview.removeEventListener("dom-ready", register); + webview.removeEventListener("render-process-gone", recoverGuest); }; - }, [config, tabId]); + }, [config, tabId, webviewGeneration]); const active = presentation.visible && presentation.rect !== null; const lastRect = presentation.rect; @@ -171,6 +180,7 @@ export function HostedBrowserWebview(props: { const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, + cornerRadius: presentation.cornerRadius, rect: lastRect, hiddenSize, }); @@ -194,8 +204,9 @@ export function HostedBrowserWebview(props: { /> ) : null} { expect( resolveBrowserSurfacePanelRect( { - hidden: { rect: staleRect, visible: false, content: null, updatedAt: 1, owner: null }, - active: { rect: liveRect, visible: true, content: null, updatedAt: 2, owner: null }, + hidden: { + rect: staleRect, + visible: false, + content: null, + cornerRadius: 0, + updatedAt: 1, + owner: null, + }, + active: { + rect: liveRect, + visible: true, + content: null, + cornerRadius: 0, + updatedAt: 2, + owner: null, + }, }, "hidden", ), diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index 58012a11a30..ffa7cf93216 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -11,6 +11,7 @@ export interface BrowserSurfacePresentation { readonly rect: BrowserSurfaceRect | null; readonly visible: boolean; readonly content: BrowserSurfaceContentPresentation | null; + readonly cornerRadius: number; readonly updatedAt: number; readonly owner: symbol | null; } @@ -33,13 +34,14 @@ interface BrowserSurfaceStoreState { owner: symbol, rect: BrowserSurfaceRect, visible: boolean, + cornerRadius: number, ) => void; readonly presentContent: (tabId: string, content: BrowserSurfaceContentPresentation) => void; readonly release: (tabId: string, owner: symbol) => void; } export interface BrowserSurfaceLease { - readonly present: (rect: BrowserSurfaceRect, visible: boolean) => void; + readonly present: (rect: BrowserSurfaceRect, visible: boolean, cornerRadius?: number) => void; readonly release: () => void; } @@ -83,21 +85,29 @@ export const useBrowserSurfaceStore = create()((set) = rect: current?.rect ?? null, visible: false, content: current?.content ?? null, + cornerRadius: current?.cornerRadius ?? 0, updatedAt: Date.now(), owner, }, }, }; }), - present: (tabId, owner, rect, visible) => + present: (tabId, owner, rect, visible, cornerRadius) => set((state) => { const current = state.byTabId[tabId]; if (current?.owner !== owner) return state; - if (current && current.visible === visible && rectEquals(current.rect, rect)) return state; + if ( + current && + current.visible === visible && + current.cornerRadius === cornerRadius && + rectEquals(current.rect, rect) + ) { + return state; + } return { byTabId: { ...state.byTabId, - [tabId]: { ...current, rect, visible, updatedAt: Date.now() }, + [tabId]: { ...current, rect, visible, cornerRadius, updatedAt: Date.now() }, }, }; }), @@ -112,6 +122,7 @@ export const useBrowserSurfaceStore = create()((set) = rect: null, visible: false, content, + cornerRadius: 0, updatedAt: Date.now(), owner: null, }, @@ -157,9 +168,9 @@ export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease { useBrowserSurfaceStore.getState().claim(tabId, owner); return { - present: (rect, visible) => { + present: (rect, visible, cornerRadius = 0) => { if (released) return; - useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible); + useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible, cornerRadius); }, release: () => { if (released) return; diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index 826684bb06f..d0298dcdee7 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -23,6 +23,23 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { }); }); + it("clips a floating webview to the mini-player frame", () => { + expect( + resolveHostedBrowserWebviewWrapperStyle({ + active: true, + cornerRadius: 12, + rect: { x: 12, y: 34, width: 360, height: 203 }, + hiddenSize: { width: 1280, height: 800 }, + }), + ).toMatchObject({ + left: 12, + top: 34, + width: 360, + height: 203, + borderRadius: 12, + }); + }); + it("keeps an inactive webview paintable while moving it offscreen", () => { const style = resolveHostedBrowserWebviewWrapperStyle({ active: false, diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index 4dade986e1f..f96f4af0462 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -12,6 +12,7 @@ export interface HostedBrowserWebviewWrapperStyle { readonly height: number; readonly zIndex: number; readonly pointerEvents: "auto" | "none"; + readonly borderRadius?: number; readonly visibility?: "visible"; } @@ -19,10 +20,11 @@ export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; export function resolveHostedBrowserWebviewWrapperStyle(input: { readonly active: boolean; + readonly cornerRadius?: number; readonly rect: BrowserSurfaceRect | null; readonly hiddenSize: HostedBrowserWebviewSize; }): HostedBrowserWebviewWrapperStyle { - const { active, hiddenSize, rect } = input; + const { active, cornerRadius = 0, hiddenSize, rect } = input; if (active && rect) { return { left: rect.x, @@ -31,6 +33,7 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { height: rect.height, zIndex: 30, pointerEvents: "auto", + ...(cornerRadius > 0 ? { borderRadius: cornerRadius } : {}), }; } diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index ab1256cddb3..918e2c92bb9 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -131,8 +131,13 @@ import { } from "../previewStateStore"; import { addBrowserSurface } from "./preview/addBrowserSurface"; import { closePreviewSession } from "./preview/closePreviewSession"; +import { ThreadPreviewMiniPlayer } from "./preview/ThreadPreviewMiniPlayer"; import { subscribePreviewAction } from "./preview/previewActionBus"; import { getConfiguredPreviewUrls } from "./preview/previewEmptyStateLogic"; +import { + selectThreadPreviewMiniPlayer, + usePreviewMiniPlayerStore, +} from "../previewMiniPlayerStore"; import { RightPanelTabs } from "./RightPanelTabs"; import { DiffWorkerPoolProvider } from "./DiffWorkerPoolProvider"; import { BranchToolbar } from "./BranchToolbar"; @@ -1484,6 +1489,9 @@ function ChatViewContent(props: ChatViewProps) { const activeFileSurface = activeRightPanelSurface?.kind === "file" ? activeRightPanelSurface : null; const activePreviewState = useThreadPreviewState(activeThreadRef); + const activePreviewMiniPlayer = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayer(state.byThreadKey, activeThreadRef), + ); const panelTerminalIds = useMemo( () => new Set( @@ -1507,6 +1515,24 @@ function ChatViewContent(props: ChatViewProps) { .reconcileBrowserSurfaces(activeThreadRef, Object.keys(activePreviewState.sessions)); }, [activePreviewState.sessions, activeThreadRef]); + useEffect(() => { + if (!activeThreadRef || !activePreviewMiniPlayer) return; + const miniTabStillExists = Boolean(activePreviewState.sessions[activePreviewMiniPlayer.tabId]); + const sameTabOpenInPanel = + previewPanelOpen && + activeRightPanelSurface?.kind === "preview" && + activeRightPanelSurface.resourceId === activePreviewMiniPlayer.tabId; + if (!miniTabStillExists || sameTabOpenInPanel) { + usePreviewMiniPlayerStore.getState().close(activeThreadRef); + } + }, [ + activePreviewMiniPlayer, + activePreviewState.sessions, + activeRightPanelSurface, + activeThreadRef, + previewPanelOpen, + ]); + const planSidebarOpen = activeRightPanelKind === "plan"; const existingOpenTerminalThreadKeys = useMemo(() => { @@ -5907,6 +5933,15 @@ function ChatViewContent(props: ChatViewProps) {
+ {activeThreadRef && activePreviewMiniPlayer ? ( + + ) : null} + diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index f4e68296972..d02b396a577 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -314,7 +314,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) if (result._tag === "Failure") { return raiseAtomCommandFailure(result); } - reconcilePreviewServerSessions(threadRef, result.value.sessions); + reconcilePreviewServerSessions(threadRef, result.value); state = readThreadPreviewState(threadRef); } tabId = request.tabId ?? state.snapshot?.tabId ?? null; diff --git a/apps/web/src/components/preview/PreviewChromeRow.tsx b/apps/web/src/components/preview/PreviewChromeRow.tsx index 4c7519e71e3..8044b472264 100644 --- a/apps/web/src/components/preview/PreviewChromeRow.tsx +++ b/apps/web/src/components/preview/PreviewChromeRow.tsx @@ -290,7 +290,7 @@ export function PreviewChromeRow({ size="icon-xs" onClick={onPictureInPicture} aria-label={ - pictureInPicture ? "Close picture-in-picture" : "Open picture-in-picture" + pictureInPicture ? "Close floating preview" : "Float preview over chat" } aria-pressed={pictureInPicture ? "true" : "false"} type="button" @@ -301,7 +301,7 @@ export function PreviewChromeRow({ - {pictureInPicture ? "Close picture-in-picture" : "Keep preview on top"} + {pictureInPicture ? "Close floating preview" : "Float preview over chat"} ) : null} diff --git a/apps/web/src/components/preview/PreviewMoreMenu.tsx b/apps/web/src/components/preview/PreviewMoreMenu.tsx index 28fc2e22232..a98d33304e8 100644 --- a/apps/web/src/components/preview/PreviewMoreMenu.tsx +++ b/apps/web/src/components/preview/PreviewMoreMenu.tsx @@ -46,6 +46,10 @@ interface Props { deviceToolbarVisible: boolean; /** Switches between fill-panel mode and a fixed responsive viewport. */ onToggleDeviceToolbar: () => void; + /** Whether the separate native always-on-top preview window is open. */ + nativePictureInPicture: boolean; + /** Toggles the optional native always-on-top preview window. */ + onNativePictureInPicture: () => void; } /** @@ -60,6 +64,8 @@ export function PreviewMoreMenu({ colorScheme, deviceToolbarVisible, onToggleDeviceToolbar, + nativePictureInPicture, + onNativePictureInPicture, }: Props) { if (!previewBridge) return null; const bridge = previewBridge; @@ -93,6 +99,11 @@ export function PreviewMoreMenu({ Open DevTools + + {nativePictureInPicture + ? "Close separate preview window" + : "Open separate preview window"} + {deviceToolbarVisible ? "Hide device toolbar" : "Show device toolbar"} diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 8f3d0fc27f4..2532b8c89ed 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -9,7 +9,12 @@ const mocks = vi.hoisted(() => ({ submittedUrl: null as ((url: string) => void) | null, emptyStateUrl: null as ((url: string) => void) | null, togglePictureInPicture: null as (() => void) | null, + toggleNativePictureInPicture: null as (() => void) | null, pictureInPicturePressed: false, + miniPlayerTabId: null as string | null, + openMiniPlayer: vi.fn(), + closeMiniPlayer: vi.fn(), + closeRightPanel: vi.fn(), openPictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), closePictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), pictureInPicture: false, @@ -96,6 +101,40 @@ vi.mock("~/browser/browserSurfaceStore", () => ({ ) => select({ byTabId: {} }), })); +vi.mock("~/previewMiniPlayerStore", () => { + const usePreviewMiniPlayerStore = Object.assign( + (select: (state: unknown) => unknown) => + select({ + byThreadKey: mocks.miniPlayerTabId + ? { + "environment-1:thread-1": { + tabId: mocks.miniPlayerTabId, + position: null, + }, + } + : {}, + }), + { + getState: () => ({ + open: mocks.openMiniPlayer, + close: mocks.closeMiniPlayer, + }), + }, + ); + return { + selectThreadPreviewMiniPlayer: ( + byThreadKey: Record, + ) => byThreadKey["environment-1:thread-1"] ?? null, + usePreviewMiniPlayerStore, + }; +}); + +vi.mock("~/rightPanelStore", () => ({ + useRightPanelStore: { + getState: () => ({ close: mocks.closeRightPanel }), + }, +})); + vi.mock("~/components/ui/toast", () => ({ stackedThreadToast: vi.fn(), toastManager: { add: vi.fn() }, @@ -116,9 +155,14 @@ vi.mock("./PreviewChromeRow", () => ({ onSubmit: (url: string) => void; onPictureInPicture?: () => void; pictureInPicture?: boolean; + trailingActions?: { + props: { onNativePictureInPicture?: () => void }; + }; }) => { mocks.submittedUrl = props.onSubmit; mocks.togglePictureInPicture = props.onPictureInPicture ?? null; + mocks.toggleNativePictureInPicture = + props.trailingActions?.props.onNativePictureInPicture ?? null; mocks.pictureInPicturePressed = props.pictureInPicture ?? false; return null; }, @@ -130,7 +174,12 @@ vi.mock("./PreviewEmptyState", () => ({ return null; }, })); -vi.mock("./PreviewMoreMenu", () => ({ PreviewMoreMenu: () => null })); +vi.mock("./PreviewMoreMenu", () => ({ + PreviewMoreMenu: (props: { onNativePictureInPicture: () => void }) => { + mocks.toggleNativePictureInPicture = props.onNativePictureInPicture; + return null; + }, +})); vi.mock("./PreviewUnreachable", () => ({ PreviewUnreachable: () => null })); vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null })); vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => null })); @@ -148,7 +197,12 @@ describe("PreviewView navigation", () => { mocks.submittedUrl = null; mocks.emptyStateUrl = null; mocks.togglePictureInPicture = null; + mocks.toggleNativePictureInPicture = null; mocks.pictureInPicturePressed = false; + mocks.miniPlayerTabId = null; + mocks.openMiniPlayer.mockClear(); + mocks.closeMiniPlayer.mockClear(); + mocks.closeRightPanel.mockClear(); mocks.openPictureInPicture.mockClear(); mocks.closePictureInPicture.mockClear(); mocks.pictureInPicture = false; @@ -217,7 +271,7 @@ describe("PreviewView navigation", () => { ); }); - it("opens and closes picture-in-picture for the active preview tab", async () => { + it("opens and closes a thread-scoped floating preview for the active tab", async () => { const props = { threadRef: { environmentId: EnvironmentId.make("environment-1"), @@ -230,12 +284,33 @@ describe("PreviewView navigation", () => { renderToStaticMarkup(); expect(mocks.pictureInPicturePressed).toBe(false); mocks.togglePictureInPicture?.(); - await vi.waitFor(() => expect(mocks.openPictureInPicture).toHaveBeenCalledWith("tab-1")); + expect(mocks.openMiniPlayer).toHaveBeenCalledWith(props.threadRef, "tab-1"); + expect(mocks.closeRightPanel).toHaveBeenCalledWith(props.threadRef); - mocks.pictureInPicture = true; + mocks.miniPlayerTabId = "tab-1"; renderToStaticMarkup(); expect(mocks.pictureInPicturePressed).toBe(true); mocks.togglePictureInPicture?.(); + expect(mocks.closeMiniPlayer).toHaveBeenCalledWith(props.threadRef); + }); + + it("keeps the native preview window as a secondary action", async () => { + const props = { + threadRef: { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }, + tabId: "tab-1", + visible: true, + } as const; + + renderToStaticMarkup(); + mocks.toggleNativePictureInPicture?.(); + await vi.waitFor(() => expect(mocks.openPictureInPicture).toHaveBeenCalledWith("tab-1")); + + mocks.pictureInPicture = true; + renderToStaticMarkup(); + mocks.toggleNativePictureInPicture?.(); await vi.waitFor(() => expect(mocks.closePictureInPicture).toHaveBeenCalledWith("tab-1")); }); }); diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index e7b7c3833c8..fd3674e429a 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -22,6 +22,8 @@ import { resolveDiscoveredServerUrl } from "~/browser/browserTargetResolver"; import { useEnvironment, useEnvironmentHttpBaseUrl } from "~/state/environments"; import { previewEnvironment } from "~/state/preview"; import { useAtomCommand } from "~/state/use-atom-command"; +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { useRightPanelStore } from "~/rightPanelStore"; import { previewBridge } from "./previewBridge"; import { subscribePreviewAction } from "./previewActionBus"; @@ -71,6 +73,9 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, const pickActiveRef = useRef(false); const isMountedRef = useRef(true); const previewState = useThreadPreviewState(threadRef); + const miniPlayer = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), + ); const addPreviewAnnotation = useComposerDraftStore((store) => store.addPreviewAnnotation); const addImage = useComposerDraftStore((store) => store.addImage); const environment = useEnvironment(threadRef.environmentId); @@ -228,6 +233,16 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, }, [url]); const handlePictureInPicture = useCallback(() => { + if (!tabId) return; + if (miniPlayer?.tabId === tabId) { + usePreviewMiniPlayerStore.getState().close(threadRef); + return; + } + usePreviewMiniPlayerStore.getState().open(threadRef, tabId); + useRightPanelStore.getState().close(threadRef); + }, [miniPlayer?.tabId, tabId, threadRef]); + + const handleNativePictureInPicture = useCallback(() => { if (!previewBridge || !tabId) return; const operation = desktopOverlay?.pictureInPicture ? previewBridge.pictureInPicture.close @@ -235,7 +250,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, void operation(tabId).catch((error) => { toastManager.add({ type: "error", - title: "Unable to update picture-in-picture", + title: "Unable to update popped-out preview", description: error instanceof Error ? error.message : "An error occurred.", }); }); @@ -602,7 +617,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, captureDisabled={!desktopOverlay || isUnreachable} recording={tabId !== null && activeRecordingTabIds.has(tabId)} onPictureInPicture={previewBridge && tabId ? handlePictureInPicture : undefined} - pictureInPicture={desktopOverlay?.pictureInPicture ?? false} + pictureInPicture={miniPlayer?.tabId === tabId} pictureInPictureDisabled={!desktopOverlay?.hasWebContents || isUnreachable} onPickElement={previewBridge && tabId ? handlePickElement : undefined} pickActive={pickActive} @@ -622,6 +637,8 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, colorScheme={desktopOverlay?.colorScheme ?? "system"} deviceToolbarVisible={viewport._tag !== "fill"} onToggleDeviceToolbar={handleToggleDeviceToolbar} + nativePictureInPicture={desktopOverlay?.pictureInPicture ?? false} + onNativePictureInPicture={handleNativePictureInPicture} /> ) : null } diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx new file mode 100644 index 00000000000..b16b3b2874c --- /dev/null +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -0,0 +1,241 @@ +"use client"; + +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { PanelRightIcon, PictureInPicture2, XIcon } from "lucide-react"; +import { + type PointerEvent as ReactPointerEvent, + useCallback, + useLayoutEffect, + useRef, +} from "react"; + +import { BrowserSurfaceSlot } from "~/browser/BrowserSurfaceSlot"; +import { Button } from "~/components/ui/button"; +import { toastManager } from "~/components/ui/toast"; +import { useThreadPreviewState } from "~/previewStateStore"; +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { useRightPanelStore } from "~/rightPanelStore"; + +import { previewBridge } from "./previewBridge"; +import { clampPreviewMiniPlayerPosition } from "./previewMiniPlayerLayout"; + +interface DragState { + readonly pointerId: number; + readonly pointerX: number; + readonly pointerY: number; + readonly playerX: number; + readonly playerY: number; +} + +interface Props { + readonly threadRef: ScopedThreadRef; + readonly tabId: string; + readonly bottomInset: number; +} + +export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props) { + const rootRef = useRef(null); + const dragRef = useRef(null); + const miniPlayer = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), + ); + const previewState = useThreadPreviewState(threadRef); + const snapshot = previewState.sessions[tabId] ?? null; + const desktopOverlay = previewState.desktopByTabId[tabId] ?? null; + const position = miniPlayer?.tabId === tabId ? miniPlayer.position : null; + const title = + snapshot?.navStatus._tag === "Idle" + ? "New tab" + : snapshot?.navStatus.title || snapshot?.navStatus.url || "Preview"; + + const close = useCallback(() => { + usePreviewMiniPlayerStore.getState().close(threadRef); + }, [threadRef]); + + const openInPanel = useCallback(() => { + usePreviewMiniPlayerStore.getState().close(threadRef); + useRightPanelStore.getState().openBrowser(threadRef, tabId); + }, [tabId, threadRef]); + + const toggleNativePictureInPicture = useCallback(() => { + if (!previewBridge) return; + const operation = desktopOverlay?.pictureInPicture + ? previewBridge.pictureInPicture.close + : previewBridge.pictureInPicture.open; + void operation(tabId).catch((error) => { + toastManager.add({ + type: "error", + title: "Unable to update popped-out preview", + description: error instanceof Error ? error.message : "An error occurred.", + }); + }); + }, [desktopOverlay?.pictureInPicture, tabId]); + + const clampAndMove = useCallback(() => { + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement)) return; + const next = clampPreviewMiniPlayerPosition( + position ?? { x: root.offsetLeft, y: root.offsetTop }, + { width: parent.clientWidth, height: parent.clientHeight }, + { width: root.offsetWidth, height: root.offsetHeight }, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); + }, [position, tabId, threadRef]); + + useLayoutEffect(() => { + clampAndMove(); + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement) || typeof ResizeObserver === "undefined") { + return; + } + const observer = new ResizeObserver(clampAndMove); + observer.observe(root); + observer.observe(parent); + return () => observer.disconnect(); + }, [clampAndMove]); + + const handlePointerDown = useCallback((event: ReactPointerEvent) => { + if (event.button !== 0) return; + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement)) return; + const rootRect = root.getBoundingClientRect(); + const parentRect = parent.getBoundingClientRect(); + dragRef.current = { + pointerId: event.pointerId, + pointerX: event.clientX, + pointerY: event.clientY, + playerX: rootRect.left - parentRect.left, + playerY: rootRect.top - parentRect.top, + }; + event.currentTarget.setPointerCapture(event.pointerId); + event.preventDefault(); + }, []); + + const handlePointerMove = useCallback( + (event: ReactPointerEvent) => { + const drag = dragRef.current; + const root = rootRef.current; + const parent = root?.offsetParent; + if ( + !drag || + drag.pointerId !== event.pointerId || + !root || + !(parent instanceof HTMLElement) + ) { + return; + } + const next = clampPreviewMiniPlayerPosition( + { + x: drag.playerX + event.clientX - drag.pointerX, + y: drag.playerY + event.clientY - drag.pointerY, + }, + { width: parent.clientWidth, height: parent.clientHeight }, + { width: root.offsetWidth, height: root.offsetHeight }, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); + }, + [tabId, threadRef], + ); + + const endDrag = useCallback((event: ReactPointerEvent) => { + if (dragRef.current?.pointerId !== event.pointerId) return; + dragRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + }, []); + + if (!snapshot || miniPlayer?.tabId !== tabId) return null; + + return ( +
+
+
+ + + +
+ + Preview · {title} + +
+ + + +
+
+ +
+
+ +
+ {!desktopOverlay?.hasWebContents ? ( +
+ Reconnecting preview… +
+ ) : null} +
+
+ ); +} diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts new file mode 100644 index 00000000000..70533ea1328 --- /dev/null +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + clampPreviewMiniPlayerPosition, + PREVIEW_MINI_PLAYER_EDGE_GAP, +} from "./previewMiniPlayerLayout"; + +describe("clampPreviewMiniPlayerPosition", () => { + it("keeps a dragged player within the chat viewport", () => { + expect( + clampPreviewMiniPlayerPosition( + { x: 900, y: -40 }, + { width: 1_000, height: 700 }, + { width: 360, height: 240 }, + ), + ).toEqual({ + x: 628, + y: PREVIEW_MINI_PLAYER_EDGE_GAP, + }); + }); + + it("keeps an edge gap when the player is larger than its container", () => { + expect( + clampPreviewMiniPlayerPosition( + { x: 20, y: 30 }, + { width: 200, height: 160 }, + { width: 360, height: 240 }, + ), + ).toEqual({ + x: PREVIEW_MINI_PLAYER_EDGE_GAP, + y: PREVIEW_MINI_PLAYER_EDGE_GAP, + }); + }); +}); diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts new file mode 100644 index 00000000000..2bf2e7d565f --- /dev/null +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -0,0 +1,27 @@ +import type { PreviewMiniPlayerPosition } from "~/previewMiniPlayerStore"; + +export interface PreviewMiniPlayerSize { + readonly width: number; + readonly height: number; +} + +export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; + +export function clampPreviewMiniPlayerPosition( + position: PreviewMiniPlayerPosition, + container: PreviewMiniPlayerSize, + player: PreviewMiniPlayerSize, +): PreviewMiniPlayerPosition { + const maxX = Math.max( + PREVIEW_MINI_PLAYER_EDGE_GAP, + container.width - player.width - PREVIEW_MINI_PLAYER_EDGE_GAP, + ); + const maxY = Math.max( + PREVIEW_MINI_PLAYER_EDGE_GAP, + container.height - player.height - PREVIEW_MINI_PLAYER_EDGE_GAP, + ); + return { + x: Math.min(Math.max(position.x, PREVIEW_MINI_PLAYER_EDGE_GAP), maxX), + y: Math.min(Math.max(position.y, PREVIEW_MINI_PLAYER_EDGE_GAP), maxY), + }; +} diff --git a/apps/web/src/components/preview/usePreviewSession.ts b/apps/web/src/components/preview/usePreviewSession.ts index 9bc2cc84c37..b2865c4eac1 100644 --- a/apps/web/src/components/preview/usePreviewSession.ts +++ b/apps/web/src/components/preview/usePreviewSession.ts @@ -2,14 +2,12 @@ import { useAtomValue } from "@effect/atom-react"; import { parseScopedThreadKey, scopedThreadKey } from "@t3tools/client-runtime/environment"; -import { runAtomCommand } from "@t3tools/client-runtime/state/runtime"; import type { ScopedThreadRef } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import { AsyncResult, Atom } from "effect/unstable/reactivity"; import { applyPreviewServerEvent, - applyPreviewServerSnapshot, readThreadPreviewState, reconcilePreviewServerSessions, } from "~/previewStateStore"; @@ -41,61 +39,26 @@ const previewSessionSyncAtom = Atom.family((threadKey: string) => { return Atom.make((get) => { let disposed = false; - let recoveryId = 0; - let recoveringUrl: string | null = null; let sessionsVersion = 0; let eventsVersion = 0; const reconcileSessions = (result: Atom.Type) => { if (!AsyncResult.isSuccess(result)) return; - if (result.value.sessions.length > 0) { - recoveringUrl = null; - recoveryId += 1; - reconcilePreviewServerSessions(threadRef, result.value.sessions); - return; - } - - const localSnapshot = readThreadPreviewState(threadRef).snapshot; - const recoverableUrl = - localSnapshot && localSnapshot.navStatus._tag !== "Idle" - ? localSnapshot.navStatus.url - : null; - if (!recoverableUrl) { - applyPreviewServerSnapshot(threadRef, null); - return; - } - if (recoveringUrl === recoverableUrl) return; - - recoveringUrl = recoverableUrl; - const currentRecoveryId = ++recoveryId; - void runAtomCommand( - get.registry, - previewEnvironment.open, - { - environmentId: threadRef.environmentId, - input: { threadId: threadRef.threadId, url: recoverableUrl }, - }, - { reportDefect: false, reportFailure: false }, - ).then((openResult) => { - if (disposed || currentRecoveryId !== recoveryId) return; - recoveringUrl = null; - if (openResult._tag === "Failure") return; - applyPreviewServerSnapshot(threadRef, openResult.value); - get.refresh(sessionsAtom); - }); + reconcilePreviewServerSessions(threadRef, result.value); }; const applyLatestEvent = (result: Atom.Type) => { if (!AsyncResult.isSuccess(result) || result.value.threadId !== threadRef.threadId) return; - applyPreviewServerEvent(threadRef, result.value); - if (result.value.type === "opened" || result.value.type === "closed") { + const currentEpoch = readThreadPreviewState(threadRef).serverEpoch; + if (currentEpoch !== null && currentEpoch !== result.value.serverEpoch) { get.refresh(sessionsAtom); + return; } + applyPreviewServerEvent(threadRef, result.value); }; get.addFinalizer(() => { disposed = true; - recoveryId += 1; }); const initialSessions = get.once(sessionsAtom); const initialEvent = get.once(eventsAtom); diff --git a/apps/web/src/previewMiniPlayerStore.test.ts b/apps/web/src/previewMiniPlayerStore.test.ts new file mode 100644 index 00000000000..d0d0e7a8a5e --- /dev/null +++ b/apps/web/src/previewMiniPlayerStore.test.ts @@ -0,0 +1,52 @@ +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { type EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { beforeEach, describe, expect, it } from "vite-plus/test"; + +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "./previewMiniPlayerStore"; + +const refA = scopeThreadRef("env-1" as EnvironmentId, ThreadId.make("thread-A")); +const refB = scopeThreadRef("env-1" as EnvironmentId, ThreadId.make("thread-B")); + +beforeEach(() => { + usePreviewMiniPlayerStore.setState({ byThreadKey: {} }); +}); + +describe("previewMiniPlayerStore", () => { + it("keeps floating previews scoped to their thread", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().open(refB, "tab-b"); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toMatchObject({ tabId: "tab-a" }); + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refB), + ).toMatchObject({ tabId: "tab-b" }); + }); + + it("preserves position when switching the floating tab within one thread", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().move(refA, "tab-a", { x: 24, y: 48 }); + usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toEqual({ + tabId: "tab-b", + position: { x: 24, y: 48 }, + }); + }); + + it("ignores stale drag updates after the floating tab changes", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + usePreviewMiniPlayerStore.getState().move(refA, "tab-a", { x: 100, y: 100 }); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toEqual({ + tabId: "tab-b", + position: null, + }); + }); +}); diff --git a/apps/web/src/previewMiniPlayerStore.ts b/apps/web/src/previewMiniPlayerStore.ts new file mode 100644 index 00000000000..5001c096bb4 --- /dev/null +++ b/apps/web/src/previewMiniPlayerStore.ts @@ -0,0 +1,75 @@ +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { create } from "zustand"; + +export interface PreviewMiniPlayerPosition { + readonly x: number; + readonly y: number; +} + +export interface PreviewMiniPlayerState { + readonly tabId: string; + readonly position: PreviewMiniPlayerPosition | null; +} + +interface PreviewMiniPlayerStoreState { + readonly byThreadKey: Record; + readonly open: (ref: ScopedThreadRef, tabId: string) => void; + readonly close: (ref: ScopedThreadRef) => void; + readonly move: (ref: ScopedThreadRef, tabId: string, position: PreviewMiniPlayerPosition) => void; + readonly removeThread: (ref: ScopedThreadRef) => void; +} + +export const usePreviewMiniPlayerStore = create()((set) => ({ + byThreadKey: {}, + open: (ref, tabId) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const current = state.byThreadKey[threadKey]; + if (current?.tabId === tabId) return state; + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { + tabId, + position: current?.position ?? null, + }, + }, + }; + }), + close: (ref) => + set((state) => { + const threadKey = scopedThreadKey(ref); + if (!(threadKey in state.byThreadKey)) return state; + const { [threadKey]: _closed, ...byThreadKey } = state.byThreadKey; + return { byThreadKey }; + }), + move: (ref, tabId, position) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const current = state.byThreadKey[threadKey]; + if (!current || current.tabId !== tabId) return state; + if (current.position?.x === position.x && current.position.y === position.y) return state; + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { ...current, position }, + }, + }; + }), + removeThread: (ref) => + set((state) => { + const threadKey = scopedThreadKey(ref); + if (!(threadKey in state.byThreadKey)) return state; + const { [threadKey]: _removed, ...byThreadKey } = state.byThreadKey; + return { byThreadKey }; + }), +})); + +export function selectThreadPreviewMiniPlayer( + byThreadKey: Record, + ref: ScopedThreadRef | null | undefined, +): PreviewMiniPlayerState | null { + if (!ref) return null; + return byThreadKey[scopedThreadKey(ref)] ?? null; +} diff --git a/apps/web/src/previewStateStore.test.ts b/apps/web/src/previewStateStore.test.ts index cc510f4da96..a4d709e2f29 100644 --- a/apps/web/src/previewStateStore.test.ts +++ b/apps/web/src/previewStateStore.test.ts @@ -1,11 +1,16 @@ import { scopedThreadKey, scopeThreadRef } from "@t3tools/client-runtime/environment"; -import { type EnvironmentId, type PreviewSessionSnapshot, ThreadId } from "@t3tools/contracts"; +import { + type EnvironmentId, + type PreviewEvent, + type PreviewSessionSnapshot, + ThreadId, +} from "@t3tools/contracts"; import { beforeEach, describe, expect, it } from "vite-plus/test"; import { __testing, applyPreviewDesktopState, - applyPreviewServerEvent, + applyPreviewServerEvent as applyPreviewServerEventImpl, applyPreviewServerSnapshot, beginPreviewSessionClose, cancelPreviewSessionClose, @@ -33,7 +38,25 @@ const makeSnapshot = (overrides: Partial = {}): PreviewS ...overrides, }); +type PreviewEventDraft = PreviewEvent extends infer Event + ? Event extends { readonly revision: number } + ? Omit + : never + : never; + +const serverEpoch = "server-a"; +let nextServerRevision = 0; +const applyPreviewServerEvent = (eventRef: typeof ref, event: PreviewEventDraft): void => { + nextServerRevision += 1; + applyPreviewServerEventImpl(eventRef, { + ...event, + serverEpoch, + revision: nextServerRevision, + } as PreviewEvent); +}; + beforeEach(() => { + nextServerRevision = 0; resetPreviewStateForTests(); }); @@ -369,7 +392,7 @@ describe("previewStateStore (single-tab)", () => { controller: "none", }); - reconcilePreviewServerSessions(ref, [active]); + reconcilePreviewServerSessions(ref, { sessions: [active], serverEpoch, revision: 1 }); const state = readThreadPreviewState(ref); expect(Object.keys(state.sessions)).toEqual([active.tabId]); @@ -381,7 +404,7 @@ describe("previewStateStore (single-tab)", () => { it("clears stale sessions when an authoritative list is empty", () => { applyPreviewServerSnapshot(ref, makeSnapshot()); - reconcilePreviewServerSessions(ref, []); + reconcilePreviewServerSessions(ref, { sessions: [], serverEpoch, revision: 1 }); const state = readThreadPreviewState(ref); expect(state.sessions).toEqual({}); @@ -389,6 +412,74 @@ describe("previewStateStore (single-tab)", () => { expect(state.snapshot).toBeNull(); }); + it("ignores a list response older than the latest server event", () => { + const snapshot = makeSnapshot(); + applyPreviewServerEvent(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + snapshot, + }); + + reconcilePreviewServerSessions(ref, { sessions: [], serverEpoch, revision: 0 }); + + expect(readThreadPreviewState(ref).sessions).toEqual({ [snapshot.tabId]: snapshot }); + }); + + it("does not resurrect a tab from an event older than its close", () => { + const snapshot = makeSnapshot(); + applyPreviewServerEvent(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + snapshot, + }); + applyPreviewServerEvent(ref, { + type: "closed", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: "2026-01-01T00:00:01.000Z", + }); + + applyPreviewServerEventImpl(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + serverEpoch, + revision: 1, + snapshot, + }); + + expect(readThreadPreviewState(ref).sessions).toEqual({}); + }); + + it("accepts a lower revision from a newly restarted server", () => { + const snapshot = makeSnapshot(); + applyPreviewServerEventImpl(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + serverEpoch, + revision: 12, + snapshot, + }); + + reconcilePreviewServerSessions(ref, { + sessions: [], + serverEpoch: "server-b", + revision: 0, + }); + + const state = readThreadPreviewState(ref); + expect(state.sessions).toEqual({}); + expect(state.serverEpoch).toBe("server-b"); + expect(state.serverRevision).toBe(0); + }); + it("applyServerSnapshot null clears snapshot for a thread that had one", () => { const snapshot = makeSnapshot(); applyPreviewServerSnapshot(ref, snapshot); diff --git a/apps/web/src/previewStateStore.ts b/apps/web/src/previewStateStore.ts index 0580b22f4fa..02b66922a43 100644 --- a/apps/web/src/previewStateStore.ts +++ b/apps/web/src/previewStateStore.ts @@ -10,6 +10,7 @@ import { scopedThreadKey } from "@t3tools/client-runtime/environment"; import { type DesktopPreviewColorScheme, type PreviewEvent, + type PreviewListResult, type PreviewSessionSnapshot, type ScopedThreadRef, } from "@t3tools/contracts"; @@ -38,6 +39,10 @@ export interface ThreadPreviewState { desktopOverlay: DesktopPreviewOverlay | null; desktopByTabId: Record; recentlySeenUrls: string[]; + /** Server process currently authoritative for revision ordering. */ + serverEpoch: string | null; + /** Latest ordered server revision applied from a list response or event. */ + serverRevision: number; } const EMPTY_THREAD_PREVIEW_STATE: ThreadPreviewState = Object.freeze({ @@ -48,6 +53,8 @@ const EMPTY_THREAD_PREVIEW_STATE: ThreadPreviewState = Object.freeze({ desktopOverlay: null, desktopByTabId: {}, recentlySeenUrls: [] as string[], + serverEpoch: null, + serverRevision: 0, }); const emptyPreviewStateAtom = Atom.make(EMPTY_THREAD_PREVIEW_STATE).pipe( @@ -177,52 +184,68 @@ export function subscribeThreadPreviewState( export function applyPreviewServerEvent(ref: ScopedThreadRef, event: PreviewEvent): void { updateThreadPreviewState(ref, (current) => { - switch (event.type) { - case "opened": - case "navigated": - case "resized": { - const snapshot = event.snapshot; - if (current.suppressedTabIds.has(snapshot.tabId)) return current; - const recentlySeenUrls = - snapshot.navStatus._tag === "Idle" - ? current.recentlySeenUrls - : dedupeRecentUrls(current.recentlySeenUrls, snapshot.navStatus.url); - const sessions = { ...current.sessions, [snapshot.tabId]: snapshot }; - const activeTabId = event.type === "opened" ? snapshot.tabId : current.activeTabId; - const activeSnapshot = sessions[activeTabId ?? snapshot.tabId] ?? snapshot; - return { - ...current, - sessions, - activeTabId: activeTabId ?? snapshot.tabId, - snapshot: activeSnapshot, - desktopOverlay: current.desktopByTabId[activeSnapshot.tabId] ?? null, - recentlySeenUrls, - }; + if (current.serverEpoch !== null && event.serverEpoch !== current.serverEpoch) return current; + if (event.revision < current.serverRevision) return current; + const next = (() => { + switch (event.type) { + case "opened": + case "navigated": + case "resized": { + const snapshot = event.snapshot; + if (current.suppressedTabIds.has(snapshot.tabId)) return current; + const recentlySeenUrls = + snapshot.navStatus._tag === "Idle" + ? current.recentlySeenUrls + : dedupeRecentUrls(current.recentlySeenUrls, snapshot.navStatus.url); + const sessions = { ...current.sessions, [snapshot.tabId]: snapshot }; + const activeTabId = event.type === "opened" ? snapshot.tabId : current.activeTabId; + const activeSnapshot = sessions[activeTabId ?? snapshot.tabId] ?? snapshot; + return { + ...current, + sessions, + activeTabId: activeTabId ?? snapshot.tabId, + snapshot: activeSnapshot, + desktopOverlay: current.desktopByTabId[activeSnapshot.tabId] ?? null, + recentlySeenUrls, + }; + } + case "failed": { + const existing = current.sessions[event.tabId]; + if (!existing) return current; + const failedSnapshot = { + ...existing, + navStatus: { + _tag: "LoadFailed" as const, + url: event.url, + title: event.title, + code: event.code, + description: event.description, + }, + updatedAt: event.createdAt, + }; + const sessions = { ...current.sessions, [event.tabId]: failedSnapshot }; + return { + ...current, + sessions, + snapshot: current.activeTabId === event.tabId ? failedSnapshot : current.snapshot, + }; + } + case "closed": { + const closed = removeSession(current, event.tabId); + if (!closed.suppressedTabIds.has(event.tabId)) return closed; + const suppressedTabIds = new Set(closed.suppressedTabIds); + suppressedTabIds.delete(event.tabId); + return { ...closed, suppressedTabIds }; + } } - case "failed": { - const existing = current.sessions[event.tabId]; - if (!existing) return current; - const failedSnapshot = { - ...existing, - navStatus: { - _tag: "LoadFailed" as const, - url: event.url, - title: event.title, - code: event.code, - description: event.description, - }, - updatedAt: event.createdAt, - }; - const sessions = { ...current.sessions, [event.tabId]: failedSnapshot }; - return { - ...current, - sessions, - snapshot: current.activeTabId === event.tabId ? failedSnapshot : current.snapshot, + })(); + return next.serverRevision === event.revision && next.serverEpoch === event.serverEpoch + ? next + : { + ...next, + serverEpoch: event.serverEpoch, + serverRevision: event.revision, }; - } - case "closed": - return removeSession(current, event.tabId); - } }); } @@ -293,9 +316,12 @@ export function updatePreviewServerSnapshot( */ export function reconcilePreviewServerSessions( ref: ScopedThreadRef, - snapshots: ReadonlyArray, + result: PreviewListResult, ): void { updateThreadPreviewState(ref, (current) => { + const sameServer = current.serverEpoch === result.serverEpoch; + if (sameServer && result.revision < current.serverRevision) return current; + const snapshots = result.sessions; const sessions: Record = {}; let recentlySeenUrls = current.recentlySeenUrls; for (const snapshot of snapshots) { @@ -315,14 +341,22 @@ export function reconcilePreviewServerSessions( const desktopByTabId = Object.fromEntries( Object.entries(current.desktopByTabId).filter(([tabId]) => sessions[tabId] !== undefined), ); + const suppressedTabIds = new Set( + [...current.suppressedTabIds].filter((tabId) => + snapshots.some((snapshot) => snapshot.tabId === tabId), + ), + ); return { ...current, sessions, + suppressedTabIds, activeTabId, snapshot, desktopByTabId, desktopOverlay: activeTabId ? (desktopByTabId[activeTabId] ?? null) : null, recentlySeenUrls, + serverEpoch: result.serverEpoch, + serverRevision: result.revision, }; }); } diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index e2e8336cda9..2b321c7eeae 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -216,6 +216,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, snapshot: { threadId: "t", tabId: "preview-t", @@ -234,6 +236,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, url: "https://example.com/", title: "", code: -105, @@ -251,6 +255,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, snapshot: { threadId: "t", tabId: "preview-t", @@ -270,6 +276,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, }); expect(event.type).toBe("closed"); }); diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index c00f878bcbe..dfc10e0b9b7 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -9,7 +9,7 @@ * @module Preview */ import { Schema } from "effect"; -import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { NonNegativeInt, PositiveInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; const Url = TrimmedNonEmptyString.check(Schema.isMaxLength(2048)); const Title = Schema.String.check(Schema.isMaxLength(512)); @@ -192,6 +192,10 @@ export type PreviewListInput = typeof PreviewListInput.Type; export const PreviewListResult = Schema.Struct({ sessions: Schema.Array(PreviewSessionSnapshot), + /** Identifies the current server process so revision resets are safe. */ + serverEpoch: TrimmedNonEmptyString, + /** Monotonic server state revision used to reject stale list responses. */ + revision: NonNegativeInt, }); export type PreviewListResult = typeof PreviewListResult.Type; @@ -199,6 +203,10 @@ const PreviewEventBaseSchema = Schema.Struct({ threadId: TrimmedNonEmptyString, tabId: PreviewTabId, createdAt: Schema.String, + /** Identifies the server process that emitted this event. */ + serverEpoch: TrimmedNonEmptyString, + /** Monotonic server state revision shared with PreviewListResult. */ + revision: PositiveInt, }); const PreviewOpenedEvent = Schema.Struct({ From 5029e1dca828f22e9aa695d6dcad8fcd657f711f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 12:43:50 +0200 Subject: [PATCH 11/26] fix(preview): harden floating preview recovery Co-authored-by: codex --- apps/web/src/browser/HostedBrowserWebview.tsx | 19 +++- .../src/browser/webviewCrashRecovery.test.ts | 41 ++++++++ apps/web/src/browser/webviewCrashRecovery.ts | 38 ++++++++ .../preview/ThreadPreviewMiniPlayer.tsx | 96 ++++++++----------- .../preview/previewMiniPlayerLayout.test.ts | 14 +++ .../preview/previewMiniPlayerLayout.ts | 4 +- 6 files changed, 155 insertions(+), 57 deletions(-) create mode 100644 apps/web/src/browser/webviewCrashRecovery.test.ts create mode 100644 apps/web/src/browser/webviewCrashRecovery.ts diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index e9bc1cb80fc..fe3f5e294ce 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -16,6 +16,11 @@ import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime import { resolveHostedBrowserWebviewWrapperStyle } from "./hostedBrowserWebviewStyle"; import { usePreviewWebviewConfig } from "./previewWebviewConfigState"; import { useBrowserViewportResize } from "./useBrowserViewportResize"; +import { + INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, + planWebviewCrashRecovery, + type WebviewCrashRecoveryState, +} from "./webviewCrashRecovery"; interface ElectronWebview extends HTMLElement { src: string; @@ -45,6 +50,7 @@ export function HostedBrowserWebview(props: { const tabLeaseRef = useRef(null); const wrapperRef = useRef(null); const webviewRef = useRef(null); + const crashRecoveryRef = useRef(INITIAL_WEBVIEW_CRASH_RECOVERY_STATE); const [aspectRatioLocked, setAspectRatioLocked] = useState(false); const presentation = useBrowserSurfaceStore( useShallow((state) => { @@ -59,6 +65,7 @@ export function HostedBrowserWebview(props: { usePreviewBridge({ threadRef, tabId }); useEffect(() => { + crashRecoveryRef.current = INITIAL_WEBVIEW_CRASH_RECOVERY_STATE; const lease = acquireDesktopTab(tabId); tabLeaseRef.current = lease; return () => { @@ -79,6 +86,7 @@ export function HostedBrowserWebview(props: { const bridge = previewBridge; if (!webview || !config || !bridge) return; let disposed = false; + let recoveryTimeout: ReturnType | null = null; const register = () => { const lease = tabLeaseRef.current; if (!lease) return; @@ -99,8 +107,14 @@ export function HostedBrowserWebview(props: { })(); }; const recoverGuest = () => { - if (disposed) return; - setWebviewGeneration((generation) => generation + 1); + if (disposed || recoveryTimeout !== null) return; + const recovery = planWebviewCrashRecovery(crashRecoveryRef.current, Date.now()); + if (!recovery) return; + crashRecoveryRef.current = recovery.state; + recoveryTimeout = setTimeout(() => { + recoveryTimeout = null; + if (!disposed) setWebviewGeneration((generation) => generation + 1); + }, recovery.delayMs); }; webview.addEventListener("did-attach", register); webview.addEventListener("dom-ready", register); @@ -108,6 +122,7 @@ export function HostedBrowserWebview(props: { register(); return () => { disposed = true; + if (recoveryTimeout !== null) clearTimeout(recoveryTimeout); webview.removeEventListener("did-attach", register); webview.removeEventListener("dom-ready", register); webview.removeEventListener("render-process-gone", recoverGuest); diff --git a/apps/web/src/browser/webviewCrashRecovery.test.ts b/apps/web/src/browser/webviewCrashRecovery.test.ts new file mode 100644 index 00000000000..a13e6ff06f6 --- /dev/null +++ b/apps/web/src/browser/webviewCrashRecovery.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, + planWebviewCrashRecovery, + WEBVIEW_CRASH_RECOVERY_WINDOW_MS, +} from "./webviewCrashRecovery"; + +describe("planWebviewCrashRecovery", () => { + it("backs off and stops after a bounded number of rapid crashes", () => { + const first = planWebviewCrashRecovery(INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, 1_000); + expect(first).not.toBeNull(); + expect(first?.delayMs).toBe(250); + + const second = planWebviewCrashRecovery(first!.state, 1_100); + expect(second).not.toBeNull(); + expect(second?.delayMs).toBe(500); + + const third = planWebviewCrashRecovery(second!.state, 1_200); + expect(third).not.toBeNull(); + expect(third?.delayMs).toBe(1_000); + + expect(planWebviewCrashRecovery(third!.state, 1_300)).toBeNull(); + }); + + it("allows recovery again after the crash window expires", () => { + const first = planWebviewCrashRecovery(INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, 1_000)!; + const second = planWebviewCrashRecovery(first.state, 1_100)!; + const third = planWebviewCrashRecovery(second.state, 1_200)!; + + expect(planWebviewCrashRecovery(third.state, 1_000 + WEBVIEW_CRASH_RECOVERY_WINDOW_MS)).toEqual( + { + delayMs: 250, + state: { + attempts: 1, + windowStartedAt: 1_000 + WEBVIEW_CRASH_RECOVERY_WINDOW_MS, + }, + }, + ); + }); +}); diff --git a/apps/web/src/browser/webviewCrashRecovery.ts b/apps/web/src/browser/webviewCrashRecovery.ts new file mode 100644 index 00000000000..2267f4a812d --- /dev/null +++ b/apps/web/src/browser/webviewCrashRecovery.ts @@ -0,0 +1,38 @@ +export const WEBVIEW_CRASH_RECOVERY_WINDOW_MS = 30_000; +export const WEBVIEW_CRASH_RECOVERY_MAX_ATTEMPTS = 3; +export const WEBVIEW_CRASH_RECOVERY_BASE_DELAY_MS = 250; + +export interface WebviewCrashRecoveryState { + readonly attempts: number; + readonly windowStartedAt: number | null; +} + +export interface WebviewCrashRecoveryPlan { + readonly delayMs: number; + readonly state: WebviewCrashRecoveryState; +} + +export const INITIAL_WEBVIEW_CRASH_RECOVERY_STATE: WebviewCrashRecoveryState = { + attempts: 0, + windowStartedAt: null, +}; + +export function planWebviewCrashRecovery( + state: WebviewCrashRecoveryState, + now: number, +): WebviewCrashRecoveryPlan | null { + const startsNewWindow = + state.windowStartedAt === null || + now - state.windowStartedAt >= WEBVIEW_CRASH_RECOVERY_WINDOW_MS; + const attempts = startsNewWindow ? 0 : state.attempts; + if (attempts >= WEBVIEW_CRASH_RECOVERY_MAX_ATTEMPTS) return null; + + const nextAttempts = attempts + 1; + return { + delayMs: WEBVIEW_CRASH_RECOVERY_BASE_DELAY_MS * 2 ** attempts, + state: { + attempts: nextAttempts, + windowStartedAt: startsNewWindow ? now : state.windowStartedAt, + }, + }; +} diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index b16b3b2874c..2f81407d8a4 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -2,12 +2,7 @@ import type { ScopedThreadRef } from "@t3tools/contracts"; import { PanelRightIcon, PictureInPicture2, XIcon } from "lucide-react"; -import { - type PointerEvent as ReactPointerEvent, - useCallback, - useLayoutEffect, - useRef, -} from "react"; +import { type PointerEvent as ReactPointerEvent, useLayoutEffect, useRef } from "react"; import { BrowserSurfaceSlot } from "~/browser/BrowserSurfaceSlot"; import { Button } from "~/components/ui/button"; @@ -48,16 +43,16 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props ? "New tab" : snapshot?.navStatus.title || snapshot?.navStatus.url || "Preview"; - const close = useCallback(() => { + const close = () => { usePreviewMiniPlayerStore.getState().close(threadRef); - }, [threadRef]); + }; - const openInPanel = useCallback(() => { + const openInPanel = () => { usePreviewMiniPlayerStore.getState().close(threadRef); useRightPanelStore.getState().openBrowser(threadRef, tabId); - }, [tabId, threadRef]); + }; - const toggleNativePictureInPicture = useCallback(() => { + const toggleNativePictureInPicture = () => { if (!previewBridge) return; const operation = desktopOverlay?.pictureInPicture ? previewBridge.pictureInPicture.close @@ -69,21 +64,21 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props description: error instanceof Error ? error.message : "An error occurred.", }); }); - }, [desktopOverlay?.pictureInPicture, tabId]); - - const clampAndMove = useCallback(() => { - const root = rootRef.current; - const parent = root?.offsetParent; - if (!root || !(parent instanceof HTMLElement)) return; - const next = clampPreviewMiniPlayerPosition( - position ?? { x: root.offsetLeft, y: root.offsetTop }, - { width: parent.clientWidth, height: parent.clientHeight }, - { width: root.offsetWidth, height: root.offsetHeight }, - ); - usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); - }, [position, tabId, threadRef]); + }; useLayoutEffect(() => { + const clampAndMove = () => { + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement)) return; + const next = clampPreviewMiniPlayerPosition( + position ?? { x: root.offsetLeft, y: root.offsetTop }, + { width: parent.clientWidth, height: parent.clientHeight }, + { width: root.offsetWidth, height: root.offsetHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); + }; clampAndMove(); const root = rootRef.current; const parent = root?.offsetParent; @@ -94,9 +89,9 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props observer.observe(root); observer.observe(parent); return () => observer.disconnect(); - }, [clampAndMove]); + }, [bottomInset, position, tabId, threadRef]); - const handlePointerDown = useCallback((event: ReactPointerEvent) => { + const handlePointerDown = (event: ReactPointerEvent) => { if (event.button !== 0) return; const root = rootRef.current; const parent = root?.offsetParent; @@ -112,41 +107,34 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props }; event.currentTarget.setPointerCapture(event.pointerId); event.preventDefault(); - }, []); + }; - const handlePointerMove = useCallback( - (event: ReactPointerEvent) => { - const drag = dragRef.current; - const root = rootRef.current; - const parent = root?.offsetParent; - if ( - !drag || - drag.pointerId !== event.pointerId || - !root || - !(parent instanceof HTMLElement) - ) { - return; - } - const next = clampPreviewMiniPlayerPosition( - { - x: drag.playerX + event.clientX - drag.pointerX, - y: drag.playerY + event.clientY - drag.pointerY, - }, - { width: parent.clientWidth, height: parent.clientHeight }, - { width: root.offsetWidth, height: root.offsetHeight }, - ); - usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); - }, - [tabId, threadRef], - ); + const handlePointerMove = (event: ReactPointerEvent) => { + const drag = dragRef.current; + const root = rootRef.current; + const parent = root?.offsetParent; + if (!drag || drag.pointerId !== event.pointerId || !root || !(parent instanceof HTMLElement)) { + return; + } + const next = clampPreviewMiniPlayerPosition( + { + x: drag.playerX + event.clientX - drag.pointerX, + y: drag.playerY + event.clientY - drag.pointerY, + }, + { width: parent.clientWidth, height: parent.clientHeight }, + { width: root.offsetWidth, height: root.offsetHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); + }; - const endDrag = useCallback((event: ReactPointerEvent) => { + const endDrag = (event: ReactPointerEvent) => { if (dragRef.current?.pointerId !== event.pointerId) return; dragRef.current = null; if (event.currentTarget.hasPointerCapture(event.pointerId)) { event.currentTarget.releasePointerCapture(event.pointerId); } - }, []); + }; if (!snapshot || miniPlayer?.tabId !== tabId) return null; diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts index 70533ea1328..762118fabc9 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts @@ -31,4 +31,18 @@ describe("clampPreviewMiniPlayerPosition", () => { y: PREVIEW_MINI_PLAYER_EDGE_GAP, }); }); + + it("keeps the player above a growing composer inset", () => { + expect( + clampPreviewMiniPlayerPosition( + { x: 500, y: 448 }, + { width: 1_000, height: 700 }, + { width: 360, height: 240 }, + 160, + ), + ).toEqual({ + x: 500, + y: 288, + }); + }); }); diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index 2bf2e7d565f..d1a04c3143e 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -11,14 +11,16 @@ export function clampPreviewMiniPlayerPosition( position: PreviewMiniPlayerPosition, container: PreviewMiniPlayerSize, player: PreviewMiniPlayerSize, + bottomInset = 0, ): PreviewMiniPlayerPosition { + const reservedBottomSpace = Math.max(0, bottomInset); const maxX = Math.max( PREVIEW_MINI_PLAYER_EDGE_GAP, container.width - player.width - PREVIEW_MINI_PLAYER_EDGE_GAP, ); const maxY = Math.max( PREVIEW_MINI_PLAYER_EDGE_GAP, - container.height - player.height - PREVIEW_MINI_PLAYER_EDGE_GAP, + container.height - reservedBottomSpace - player.height - PREVIEW_MINI_PLAYER_EDGE_GAP, ); return { x: Math.min(Math.max(position.x, PREVIEW_MINI_PLAYER_EDGE_GAP), maxX), From 3505d3186aa3d16fddacc48c35cd0b3354ab180c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 13:08:08 +0200 Subject: [PATCH 12/26] fix(preview): harden capture and tab lifecycle races Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 85 ++++++++-- apps/desktop/src/preview/Manager.ts | 160 ++++++++++-------- apps/server/src/preview/Manager.test.ts | 20 +++ apps/server/src/preview/Manager.ts | 3 +- apps/web/src/browser/browserRecording.test.ts | 12 +- apps/web/src/browser/browserRecording.ts | 4 + .../src/browser/browserRecordingScope.test.ts | 43 +++++ apps/web/src/browser/browserRecordingScope.ts | 14 ++ .../preview/PreviewAutomationHosts.tsx | 21 ++- 9 files changed, 275 insertions(+), 87 deletions(-) create mode 100644 apps/web/src/browser/browserRecordingScope.test.ts create mode 100644 apps/web/src/browser/browserRecordingScope.ts diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 9fbc632f052..0d1c4f91e5d 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -510,7 +510,7 @@ describe("PreviewManager", () => { ), ); - effectIt.effect("detaches a webview registered while tab close cleanup is in flight", () => + effectIt.effect("blocks late webview and capture starts during tab close", () => withManager((manager) => Effect.gen(function* () { const capturePage = vi.fn(async () => ({ @@ -520,6 +520,7 @@ describe("PreviewManager", () => { const firstWebContents = makeTestPreviewWebContents(capturePage, 42); const replacementWebContents = makeTestPreviewWebContents(capturePage, 43); const replacementListenerSpies = replacementWebContents as unknown as { + readonly on: ReturnType; readonly off: ReturnType; readonly ipc: { readonly off: ReturnType }; }; @@ -551,18 +552,25 @@ describe("PreviewManager", () => { .closeTab("tab_close_register_race") .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(closeCleanupPaused); - yield* manager.registerWebview("tab_close_register_race", 43); + const registrationExit = yield* Effect.exit( + manager.registerWebview("tab_close_register_race", 43), + ); + const recordingExit = yield* Effect.exit(manager.startRecording("tab_close_register_race")); yield* Deferred.succeed(continueCloseCleanup, undefined); yield* Fiber.join(closeFiber); - expect(replacementListenerSpies.off).toHaveBeenCalledWith( - "did-navigate", - expect.any(Function), - ); - expect(replacementListenerSpies.ipc.off).toHaveBeenCalledWith( - "preview:human-input", - expect.any(Function), - ); + for (const exit of [registrationExit, recordingExit]) { + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isSuccess(exit)) continue; + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewTabNotFoundError", + tabId: "tab_close_register_race", + }); + } + expect(replacementListenerSpies.on).not.toHaveBeenCalled(); + expect(replacementListenerSpies.off).not.toHaveBeenCalled(); + expect(replacementListenerSpies.ipc.off).not.toHaveBeenCalled(); + expect(capturePage).toHaveBeenCalledOnce(); }), ), ); @@ -835,6 +843,63 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("drops a captured frame when the tab webview changes during capture", () => + withManager((manager) => + Effect.gen(function* () { + const staleImage: TestCapturedPreviewImage = { + toJPEG: vi.fn(() => Buffer.from("stale-recording-frame")), + getSize: vi.fn(() => ({ width: 1280, height: 720 })), + }; + let markCaptureStarted!: () => void; + const captureStarted = new Promise((resolve) => { + markCaptureStarted = resolve; + }); + let resolveCapture: ((image: TestCapturedPreviewImage) => void) | undefined; + const staleCapturePage = vi.fn(() => { + markCaptureStarted(); + return new Promise((resolve) => { + resolveCapture = resolve; + }); + }); + const replacementCapturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("replacement-recording-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const initialWebContents = makeTestPreviewWebContents(staleCapturePage, 42); + const replacementWebContents = makeTestPreviewWebContents(replacementCapturePage, 43); + fromId.mockImplementation((webContentsId?: number) => { + if (webContentsId === 42) return initialWebContents; + if (webContentsId === 43) return replacementWebContents; + return null; + }); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_capture_replaced"); + yield* manager.registerWebview("tab_capture_replaced", 42); + const recordingFiber = yield* manager + .startRecording("tab_capture_replaced") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => captureStarted); + + yield* manager.registerWebview("tab_capture_replaced", 43); + resolveCapture?.(staleImage); + yield* Fiber.join(recordingFiber); + + expect(staleImage.getSize).not.toHaveBeenCalled(); + expect(staleImage.toJPEG).not.toHaveBeenCalled(); + expect(frames).toHaveLength(0); + expect(replacementCapturePage).not.toHaveBeenCalled(); + + yield* manager.stopRecording("tab_capture_replaced"); + }), + ), + ); + effectIt.effect("shares background frame capture between recording and picture-in-picture", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 12a26c12017..b3b3965e9a7 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -470,6 +470,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function >(new Map()); const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); + const closingTabIdsRef = yield* Ref.make>(new Set()); const attempt = (errorContext: PreviewOperationContext, evaluate: () => A) => Effect.try({ @@ -1349,50 +1350,66 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { - if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; - yield* Effect.all( - [ - cancelPickElement(tabId), - closePictureInPicture(tabId), - stopFrameCapture(tabId, "recording"), - ], - { - concurrency: 3, - discard: true, - }, - ); - const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { - const current = tabs.get(tabId); - if (!current) return [Option.none(), tabs] as const; - return [ - Option.some(current), - replaceMap(tabs, (copy) => { - copy.delete(tabId); - }), - ] as const; + const claimed = yield* Ref.modify(closingTabIdsRef, (closingTabIds) => { + if (closingTabIds.has(tabId)) return [false, closingTabIds] as const; + return [true, new Set([...closingTabIds, tabId])] as const; }); - if (Option.isNone(tab)) return; - const closedTab = tab.value; - if (closedTab.webContentsId != null) { + if (!claimed) return; + return yield* Effect.gen(function* () { + if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; yield* Effect.all( - [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], - { concurrency: 2, discard: true }, + [ + cancelPickElement(tabId), + closePictureInPicture(tabId), + stopFrameCapture(tabId, "recording"), + ], + { + concurrency: 3, + discard: true, + }, ); - } - const updatedAt = yield* currentIso; - const closed: PreviewTabState = { - ...closedTab, - webContentsId: null, - navStatus: { kind: "Idle" }, - canGoBack: false, - canGoForward: false, - zoomFactor: DEFAULT_ZOOM_FACTOR, - pictureInPicture: false, - colorScheme: "system", - controller: "none", - updatedAt, - }; - yield* emit(tabId, closed); + const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if (!current) return [Option.none(), tabs] as const; + return [ + Option.some(current), + replaceMap(tabs, (copy) => { + copy.delete(tabId); + }), + ] as const; + }); + if (Option.isNone(tab)) return; + const closedTab = tab.value; + if (closedTab.webContentsId != null) { + yield* Effect.all( + [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], + { concurrency: 2, discard: true }, + ); + } + const updatedAt = yield* currentIso; + const closed: PreviewTabState = { + ...closedTab, + webContentsId: null, + navStatus: { kind: "Idle" }, + canGoBack: false, + canGoForward: false, + zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, + colorScheme: "system", + controller: "none", + updatedAt, + }; + yield* emit(tabId, closed); + }).pipe( + Effect.ensuring( + Ref.update(closingTabIdsRef, (closingTabIds) => { + if (!closingTabIds.has(tabId)) return closingTabIds; + const next = new Set(closingTabIds); + next.delete(tabId); + return next; + }), + ), + ); }); const registerWebview = Effect.fn("PreviewManager.registerWebview")(function* ( @@ -1400,7 +1417,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function webContentsId: number, ) { const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (!tab) { + if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { return yield* new PreviewTabNotFoundError({ tabId }); } const wc = webContents.fromId(webContentsId); @@ -1894,6 +1911,18 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, () => wc.capturePage(), ); + const captureStillCurrent = yield* Effect.all( + [SynchronizedRef.get(frameCaptureSessionsRef), SynchronizedRef.get(tabsRef)], + { concurrency: 2 }, + ).pipe( + Effect.map( + ([captureSessions, tabs]) => + captureSessions.get(tabId) === captureSession && + tabs.get(tabId)?.webContentsId === wc.id && + !wc.isDestroyed(), + ), + ); + if (!captureStillCurrent) return; const size = yield* attempt( { operation: "frameCapture.measureFrame", @@ -2009,22 +2038,26 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ); const created = yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => { - const current = sessions.get(tabId); - if (current) { - if (current.consumers.has(consumer)) { - return Effect.succeed([false, sessions] as const); - } - return Effect.succeed([ - false, - replaceMap(sessions, (copy) => { - copy.set(tabId, { - ...current, - consumers: new Set([...current.consumers, consumer]), - }); - }), - ] as const); - } return Effect.gen(function* () { + const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { + return yield* new PreviewTabNotFoundError({ tabId }); + } + const current = sessions.get(tabId); + if (current) { + if (current.consumers.has(consumer)) { + return [false, sessions] as const; + } + return [ + false, + replaceMap(sessions, (copy) => { + copy.set(tabId, { + ...current, + consumers: new Set([...current.consumers, consumer]), + }); + }), + ] as const; + } const scope = yield* Scope.fork(parentScope, "sequential"); yield* Effect.forkIn(Effect.forever(captureNextFrame), scope); return [ @@ -3171,18 +3204,6 @@ export class PreviewArtifactImageLoadError extends Schema.TaggedErrorClass()( - "PreviewRecordingAlreadyActiveError", - { - requestedTabId: Schema.String, - activeTabId: Schema.String, - }, -) { - override get message(): string { - return `Cannot record preview tab ${this.requestedTabId} while tab ${this.activeTabId} is already recording`; - } -} - export class PreviewAutomationDevToolsOpenError extends Schema.TaggedErrorClass()( "PreviewAutomationDevToolsOpenError", { webContentsId: Schema.Number }, @@ -3345,7 +3366,6 @@ export const PreviewManagerError = Schema.Union([ PreviewOperationError, PreviewArtifactPathOutsideDirectoryError, PreviewArtifactImageLoadError, - PreviewRecordingAlreadyActiveError, PreviewAutomationDevToolsOpenError, PreviewAutomationDebuggerAttachedError, PreviewAutomationEvaluationError, diff --git a/apps/server/src/preview/Manager.test.ts b/apps/server/src/preview/Manager.test.ts index ff662cdaae4..8b3dabfa338 100644 --- a/apps/server/src/preview/Manager.test.ts +++ b/apps/server/src/preview/Manager.test.ts @@ -280,6 +280,26 @@ it.layer(PreviewManager.layer)("PreviewManager", (it) => { }), ); + it.effect("gives every tab in a batch close its own monotonic revision", () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + yield* manager.open({ threadId, url: "http://localhost:5173" }); + yield* manager.open({ threadId, url: "http://localhost:3000" }); + const collector = yield* collectEvents; + + yield* manager.close({ threadId }); + + const events = yield* collector.drain; + const listed = yield* manager.list({ threadId }); + expect(events).toHaveLength(2); + expect(events.every((event) => event.type === "closed")).toBe(true); + expect(events[1]!.revision).toBeGreaterThan(events[0]!.revision); + expect(listed.revision).toBe(events[1]!.revision); + expect(listed.sessions).toHaveLength(0); + }), + ); + it.effect("close is idempotent for unknown threads", () => Effect.gen(function* () { const threadId = freshThreadId(); diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index c3487f6321b..e38e28ecd2e 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -379,8 +379,9 @@ export const make = Effect.gen(function* PreviewManagerMake() { (entry): entry is PreviewSessionState => entry !== undefined, ) : sessionsForThread(state, input.threadId); - const revision = targets.length > 0 ? state.revision + 1 : state.revision; + let revision = state.revision; for (const target of targets) { + revision += 1; sessions.delete(compositeKey(target.threadId, target.tabId)); eventsToEmit.push({ type: "closed", diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 4fd2cde5db7..36d0bf18e29 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -89,6 +89,7 @@ import { BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS, BrowserRecordingConflictError, BrowserRecordingOperationError, + readActiveBrowserRecordingTabIds, startBrowserRecording, stopBrowserRecording, } from "./browserRecording"; @@ -331,11 +332,14 @@ describe("browser recording", () => { expect(startScreencast).toHaveBeenCalledTimes(2); expect(onFrame).toHaveBeenCalledOnce(); expect(events).toContain("publish:recording-tab,recording-tab-2"); + expect(readActiveBrowserRecordingTabIds()).toEqual( + new Set(["recording-tab", "recording-tab-2"]), + ); - await Promise.all([ - stopBrowserRecording("recording-tab"), - stopBrowserRecording("recording-tab-2"), - ]); + await stopBrowserRecording("recording-tab"); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set(["recording-tab-2"])); + await stopBrowserRecording("recording-tab-2"); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); expect(save).toHaveBeenCalledTimes(2); }); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 63c67800687..603762c8b6a 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -115,6 +115,10 @@ let unsubscribeFrames: (() => void) | null = null; export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000; export const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 5_000; +export function readActiveBrowserRecordingTabIds(): ReadonlySet { + return new Set(activeRecordings.keys()); +} + const preferredMimeType = (): string => { const candidates = ["video/mp4;codecs=avc1.42E01E", "video/webm;codecs=vp9", "video/webm"]; return candidates.find((candidate) => MediaRecorder.isTypeSupported(candidate)) ?? "video/webm"; diff --git a/apps/web/src/browser/browserRecordingScope.test.ts b/apps/web/src/browser/browserRecordingScope.test.ts new file mode 100644 index 00000000000..304adf4f19e --- /dev/null +++ b/apps/web/src/browser/browserRecordingScope.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { resolveBrowserRecordingStopTarget } from "./browserRecordingScope"; + +describe("resolveBrowserRecordingStopTarget", () => { + it("stops the only active recording when the implicit browser target changed", () => { + expect(resolveBrowserRecordingStopTarget(new Set(["tab-recording"]), "tab-browsing")).toBe( + "tab-recording", + ); + }); + + it("prefers an implicit target that is actively recording", () => { + expect( + resolveBrowserRecordingStopTarget( + new Set(["tab-recording-a", "tab-recording-b"]), + "tab-recording-b", + ), + ).toBe("tab-recording-b"); + }); + + it("does not guess when multiple recordings are active and the implicit target is not one", () => { + expect( + resolveBrowserRecordingStopTarget( + new Set(["tab-recording-a", "tab-recording-b"]), + "tab-browsing", + ), + ).toBeNull(); + }); + + it("only stops an explicitly requested tab when that tab is recording", () => { + const activeTabIds = new Set(["tab-recording"]); + expect(resolveBrowserRecordingStopTarget(activeTabIds, "tab-browsing", "tab-recording")).toBe( + "tab-recording", + ); + expect(resolveBrowserRecordingStopTarget(activeTabIds, "tab-recording", "tab-browsing")).toBe( + null, + ); + }); + + it("returns null when no matching recording is active", () => { + expect(resolveBrowserRecordingStopTarget(new Set(), "tab-browsing")).toBeNull(); + }); +}); diff --git a/apps/web/src/browser/browserRecordingScope.ts b/apps/web/src/browser/browserRecordingScope.ts new file mode 100644 index 00000000000..ffbcc19a545 --- /dev/null +++ b/apps/web/src/browser/browserRecordingScope.ts @@ -0,0 +1,14 @@ +export function resolveBrowserRecordingStopTarget( + activeTabIds: ReadonlySet, + implicitTabId: string | null, + explicitTabId?: string, +): string | null { + if (explicitTabId !== undefined) { + return activeTabIds.has(explicitTabId) ? explicitTabId : null; + } + if (implicitTabId !== null && activeTabIds.has(implicitTabId)) { + return implicitTabId; + } + if (activeTabIds.size !== 1) return null; + return activeTabIds.values().next().value ?? null; +} diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index d02b396a577..9e861bf3090 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -31,7 +31,12 @@ import { } from "~/previewStateStore"; import { useRightPanelStore } from "~/rightPanelStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; -import { startBrowserRecording, stopBrowserRecording } from "~/browser/browserRecording"; +import { + readActiveBrowserRecordingTabIds, + startBrowserRecording, + stopBrowserRecording, +} from "~/browser/browserRecording"; +import { resolveBrowserRecordingStopTarget } from "~/browser/browserRecordingScope"; import { useBrowserSurfaceStore } from "~/browser/browserSurfaceStore"; import { isElectron } from "~/env"; import { useEnvironments } from "~/state/environments"; @@ -513,7 +518,19 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; } case "recordingStop": { - const artifact = tabId ? await stopBrowserRecording(tabId) : null; + const threadTabIds = new Set(Object.keys(state.sessions)); + const activeTabIds = new Set( + Array.from(readActiveBrowserRecordingTabIds()).filter((activeTabId) => + threadTabIds.has(activeTabId), + ), + ); + const stopTabId = resolveBrowserRecordingStopTarget( + activeTabIds, + tabId, + request.tabIdExplicit ? request.tabId : undefined, + ); + tabId = stopTabId ?? tabId; + const artifact = stopTabId ? await stopBrowserRecording(stopTabId) : null; if (!artifact) { return raisePreviewAutomationHostError( new PreviewAutomationRecordingNotActiveError({ From a272ca90c79999e7884baa173b3a9b9762d016e8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 13:23:55 +0200 Subject: [PATCH 13/26] fix(preview): serialize teardown ownership Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 9 +- apps/desktop/src/preview/Manager.ts | 95 +++++++++++++------ apps/web/src/browser/browserRecording.test.ts | 15 ++- apps/web/src/browser/browserRecording.ts | 22 ++++- .../preview/PreviewAutomationHosts.tsx | 9 +- .../src/components/preview/PreviewView.tsx | 4 +- 6 files changed, 109 insertions(+), 45 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 0d1c4f91e5d..da091008a9d 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -552,12 +552,15 @@ describe("PreviewManager", () => { .closeTab("tab_close_register_race") .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(closeCleanupPaused); - const registrationExit = yield* Effect.exit( - manager.registerWebview("tab_close_register_race", 43), - ); + const registrationFiber = yield* manager + .registerWebview("tab_close_register_race", 43) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(replacementListenerSpies.on).not.toHaveBeenCalled(); const recordingExit = yield* Effect.exit(manager.startRecording("tab_close_register_race")); yield* Deferred.succeed(continueCloseCleanup, undefined); yield* Fiber.join(closeFiber); + const registrationExit = yield* Fiber.await(registrationFiber); for (const exit of [registrationExit, recordingExit]) { expect(Exit.isFailure(exit)).toBe(true); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index b3b3965e9a7..007824c0139 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -471,6 +471,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); const closingTabIdsRef = yield* Ref.make>(new Set()); + const tabLifecycleLocks = new Map< + string, + { readonly semaphore: Semaphore.Semaphore; users: number } + >(); const attempt = (errorContext: PreviewOperationContext, evaluate: () => A) => Effect.try({ @@ -501,6 +505,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function update(copy); return copy; }; + const withTabLifecycleLock = ( + tabId: string, + effect: Effect.Effect, + ): Effect.Effect => + Effect.suspend(() => { + const lifecycle = tabLifecycleLocks.get(tabId) ?? { + semaphore: Semaphore.makeUnsafe(1), + users: 0, + }; + lifecycle.users += 1; + tabLifecycleLocks.set(tabId, lifecycle); + return lifecycle.semaphore.withPermit(effect).pipe( + Effect.ensuring( + Effect.sync(() => { + lifecycle.users -= 1; + if (lifecycle.users === 0 && tabLifecycleLocks.get(tabId) === lifecycle) { + tabLifecycleLocks.delete(tabId); + } + }), + ), + ); + }); const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* ( tabId: string, consumer: FrameCaptureConsumer, @@ -1349,7 +1375,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return state; }); - const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { + const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { const claimed = yield* Ref.modify(closingTabIdsRef, (closingTabIds) => { if (closingTabIds.has(tabId)) return [false, closingTabIds] as const; return [true, new Set([...closingTabIds, tabId])] as const; @@ -1412,7 +1438,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); - const registerWebview = Effect.fn("PreviewManager.registerWebview")(function* ( + const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { + return yield* withTabLifecycleLock(tabId, closeTabUnlocked(tabId)); + }); + + const registerWebviewUnlocked = Effect.fn("PreviewManager.registerWebviewUnlocked")(function* ( tabId: string, webContentsId: number, ) { @@ -1468,34 +1498,36 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); yield* attachListeners(tabId, wc); const registeredAt = yield* currentIso; - const registration = yield* SynchronizedRef.modify(tabsRef, (tabs) => { - const current = tabs.get(tabId); - if (!current) { + const registration = yield* SynchronizedRef.modifyEffect(tabsRef, (tabs) => + Effect.gen(function* () { + const current = tabs.get(tabId); + if (!current || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { + return [ + Option.none<{ readonly state: PreviewTabState; readonly pendingUrl: string | null }>(), + tabs, + ] as const; + } + const pendingUrl = current.navStatus.kind === "Loading" ? current.navStatus.url : null; + const next: PreviewTabState = { + ...current, + webContentsId, + navStatus: pendingUrl === null ? computeNavStatus(wc) : current.navStatus, + canGoBack: wc.navigationHistory.canGoBack(), + canGoForward: wc.navigationHistory.canGoForward(), + zoomFactor, + updatedAt: registeredAt, + }; return [ - Option.none<{ readonly state: PreviewTabState; readonly pendingUrl: string | null }>(), - tabs, + Option.some({ + state: next, + pendingUrl, + }), + replaceMap(tabs, (copy) => { + copy.set(tabId, next); + }), ] as const; - } - const pendingUrl = current.navStatus.kind === "Loading" ? current.navStatus.url : null; - const next: PreviewTabState = { - ...current, - webContentsId, - navStatus: pendingUrl === null ? computeNavStatus(wc) : current.navStatus, - canGoBack: wc.navigationHistory.canGoBack(), - canGoForward: wc.navigationHistory.canGoForward(), - zoomFactor, - updatedAt: registeredAt, - }; - return [ - Option.some({ - state: next, - pendingUrl, - }), - replaceMap(tabs, (copy) => { - copy.set(tabId, next); - }), - ] as const; - }); + }), + ); if (Option.isNone(registration)) { yield* Effect.all([detachControlSession(webContentsId), detachListeners(webContentsId)], { concurrency: 2, @@ -1524,6 +1556,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } }); + const registerWebview = Effect.fn("PreviewManager.registerWebview")(function* ( + tabId: string, + webContentsId: number, + ) { + return yield* withTabLifecycleLock(tabId, registerWebviewUnlocked(tabId, webContentsId)); + }); + const navigate = Effect.fn("PreviewManager.navigate")(function* (tabId: string, rawUrl: string) { const url = yield* attempt({ operation: "navigate.normalizeUrl", tabId }, () => normalizePreviewUrl(rawUrl), diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index 36d0bf18e29..962dc4b100b 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -1,3 +1,4 @@ +import { EnvironmentId, ThreadId } from "@t3tools/contracts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; const { @@ -315,6 +316,14 @@ describe("browser recording", () => { }); it("records separate tabs concurrently", async () => { + const firstThreadRef = { + environmentId: EnvironmentId.make("environment-recording"), + threadId: ThreadId.make("thread-recording-first"), + }; + const secondThreadRef = { + environmentId: EnvironmentId.make("environment-recording"), + threadId: ThreadId.make("thread-recording-second"), + }; surfaceState.byTabId = { ...surfaceState.byTabId, "recording-tab-2": { @@ -325,8 +334,8 @@ describe("browser recording", () => { }; await Promise.all([ - startBrowserRecording("recording-tab"), - startBrowserRecording("recording-tab-2"), + startBrowserRecording("recording-tab", firstThreadRef), + startBrowserRecording("recording-tab-2", secondThreadRef), ]); expect(startScreencast).toHaveBeenCalledTimes(2); @@ -335,6 +344,8 @@ describe("browser recording", () => { expect(readActiveBrowserRecordingTabIds()).toEqual( new Set(["recording-tab", "recording-tab-2"]), ); + expect(readActiveBrowserRecordingTabIds(firstThreadRef)).toEqual(new Set(["recording-tab"])); + expect(readActiveBrowserRecordingTabIds(secondThreadRef)).toEqual(new Set(["recording-tab-2"])); await stopBrowserRecording("recording-tab"); expect(readActiveBrowserRecordingTabIds()).toEqual(new Set(["recording-tab-2"])); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 603762c8b6a..56211e95351 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -1,6 +1,7 @@ import type { DesktopPreviewRecordingArtifact, DesktopPreviewRecordingFrame, + ScopedThreadRef, } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import * as Schema from "effect/Schema"; @@ -82,6 +83,7 @@ type BrowserRecordingLifecycle = interface ActiveRecording { readonly tabId: string; + readonly threadRef: ScopedThreadRef | null; readonly canvas: HTMLCanvasElement; readonly context: CanvasRenderingContext2D; readonly chunks: Blob[]; @@ -115,8 +117,18 @@ let unsubscribeFrames: (() => void) | null = null; export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000; export const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 5_000; -export function readActiveBrowserRecordingTabIds(): ReadonlySet { - return new Set(activeRecordings.keys()); +export function readActiveBrowserRecordingTabIds(threadRef?: ScopedThreadRef): ReadonlySet { + const tabIds = new Set(); + for (const recording of activeRecordings.values()) { + if ( + threadRef === undefined || + (recording.threadRef?.environmentId === threadRef.environmentId && + recording.threadRef.threadId === threadRef.threadId) + ) { + tabIds.add(recording.tabId); + } + } + return tabIds; } const preferredMimeType = (): string => { @@ -268,7 +280,10 @@ const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Prom const isStartupWaitTimeout = (error: unknown): error is BrowserRecordingOperationError => isBrowserRecordingOperationError(error) && error.operation === "wait-startup"; -export async function startBrowserRecording(tabId: string): Promise { +export async function startBrowserRecording( + tabId: string, + threadRef: ScopedThreadRef | null = null, +): Promise { const bridge = previewBridge; if (!bridge) throw new BrowserRecordingUnavailableError({ tabId }); const activeRecording = activeRecordings.get(tabId); @@ -306,6 +321,7 @@ export async function startBrowserRecording(tabId: string): Promise { }); const recording: ActiveRecording = { tabId, + threadRef, canvas, context, chunks, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 9e861bf3090..e38b91227a6 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -510,7 +510,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "recordingStart": { const ready = await requireReadyTab(); - const startedAt = await startBrowserRecording(ready.tabId); + const startedAt = await startBrowserRecording(ready.tabId, threadRef); return { tabId: ready.tabId, recording: true, @@ -518,12 +518,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; } case "recordingStop": { - const threadTabIds = new Set(Object.keys(state.sessions)); - const activeTabIds = new Set( - Array.from(readActiveBrowserRecordingTabIds()).filter((activeTabId) => - threadTabIds.has(activeTabId), - ), - ); + const activeTabIds = readActiveBrowserRecordingTabIds(threadRef); const stopTabId = resolveBrowserRecordingStopTarget( activeTabIds, tabId, diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index fd3674e429a..db72111d1cc 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -354,7 +354,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, return; } if (record) { - void startBrowserRecording(tabId).catch((error) => { + void startBrowserRecording(tabId, threadRef).catch((error) => { toastManager.add({ type: "error", title: "Unable to start recording", @@ -493,7 +493,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, }, ); }, - [activeRecordingTabIds, tabId], + [activeRecordingTabIds, tabId, threadRef], ); const handlePickElement = useCallback(() => { From 606bf3d7929ef2d0e6f64d747fa8a914160b1baf Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 13:34:11 +0200 Subject: [PATCH 14/26] fix(preview): isolate tab lifecycle generations Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 6 + apps/desktop/src/preview/Manager.ts | 177 +++++++++++++---------- 2 files changed, 109 insertions(+), 74 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index da091008a9d..4dc3675a58a 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -552,14 +552,19 @@ describe("PreviewManager", () => { .closeTab("tab_close_register_race") .pipe(Effect.forkChild({ startImmediately: true })); yield* Deferred.await(closeCleanupPaused); + const recreateFiber = yield* manager + .createTab("tab_close_register_race") + .pipe(Effect.forkChild({ startImmediately: true })); const registrationFiber = yield* manager .registerWebview("tab_close_register_race", 43) .pipe(Effect.forkChild({ startImmediately: true })); yield* Effect.yieldNow; expect(replacementListenerSpies.on).not.toHaveBeenCalled(); + yield* manager.closeTab("tab_close_register_race"); const recordingExit = yield* Effect.exit(manager.startRecording("tab_close_register_race")); yield* Deferred.succeed(continueCloseCleanup, undefined); yield* Fiber.join(closeFiber); + const recreated = yield* Fiber.join(recreateFiber); const registrationExit = yield* Fiber.await(registrationFiber); for (const exit of [registrationExit, recordingExit]) { @@ -574,6 +579,7 @@ describe("PreviewManager", () => { expect(replacementListenerSpies.off).not.toHaveBeenCalled(); expect(replacementListenerSpies.ipc.off).not.toHaveBeenCalled(); expect(capturePage).toHaveBeenCalledOnce(); + expect(recreated.webContentsId).toBeNull(); }), ), ); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 007824c0139..28b950ea5c3 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -475,6 +475,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function string, { readonly semaphore: Semaphore.Semaphore; users: number } >(); + const tabLifecycleGenerations = new Map(); const attempt = (errorContext: PreviewOperationContext, evaluate: () => A) => Effect.try({ @@ -1347,86 +1348,105 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); }); - const createTab = Effect.fn("PreviewManager.createTab")(function* (tabId: string) { + const createTabUnlocked = Effect.fn("PreviewManager.createTabUnlocked")(function* ( + tabId: string, + ) { const updatedAt = yield* currentIso; - const state = yield* SynchronizedRef.modify(tabsRef, (tabs) => { - const existing = tabs.get(tabId); - if (existing) return [existing, tabs] as const; - const initial: PreviewTabState = { - tabId, - webContentsId: null, - navStatus: { kind: "Idle" }, - canGoBack: false, - canGoForward: false, - zoomFactor: DEFAULT_ZOOM_FACTOR, - pictureInPicture: false, - colorScheme: "system", - controller: "none", - updatedAt, - }; + const result = yield* SynchronizedRef.modify( + tabsRef, + ( + tabs, + ): readonly [ + { readonly state: PreviewTabState; readonly created: boolean }, + ReadonlyMap, + ] => { + const existing = tabs.get(tabId); + if (existing) return [{ state: existing, created: false }, tabs] as const; + const initial: PreviewTabState = { + tabId, + webContentsId: null, + navStatus: { kind: "Idle" }, + canGoBack: false, + canGoForward: false, + zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, + colorScheme: "system", + controller: "none", + updatedAt, + }; + return [ + { state: initial, created: true }, + replaceMap(tabs, (copy) => { + copy.set(tabId, initial); + }), + ] as const; + }, + ); + if (result.created) { + tabLifecycleGenerations.set(tabId, (tabLifecycleGenerations.get(tabId) ?? 0) + 1); + } + yield* emit(tabId, result.state); + return result.state; + }); + + const createTab = Effect.fn("PreviewManager.createTab")(function* (tabId: string) { + return yield* withTabLifecycleLock(tabId, createTabUnlocked(tabId)); + }); + + const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { + if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; + yield* Effect.all( + [ + cancelPickElement(tabId), + closePictureInPicture(tabId), + stopFrameCapture(tabId, "recording"), + ], + { + concurrency: 3, + discard: true, + }, + ); + const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if (!current) return [Option.none(), tabs] as const; return [ - initial, + Option.some(current), replaceMap(tabs, (copy) => { - copy.set(tabId, initial); + copy.delete(tabId); }), ] as const; }); - yield* emit(tabId, state); - return state; + if (Option.isNone(tab)) return; + const closedTab = tab.value; + if (closedTab.webContentsId != null) { + yield* Effect.all( + [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], + { concurrency: 2, discard: true }, + ); + } + const updatedAt = yield* currentIso; + const closed: PreviewTabState = { + ...closedTab, + webContentsId: null, + navStatus: { kind: "Idle" }, + canGoBack: false, + canGoForward: false, + zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, + colorScheme: "system", + controller: "none", + updatedAt, + }; + yield* emit(tabId, closed); }); - const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { + const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { const claimed = yield* Ref.modify(closingTabIdsRef, (closingTabIds) => { if (closingTabIds.has(tabId)) return [false, closingTabIds] as const; return [true, new Set([...closingTabIds, tabId])] as const; }); if (!claimed) return; - return yield* Effect.gen(function* () { - if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; - yield* Effect.all( - [ - cancelPickElement(tabId), - closePictureInPicture(tabId), - stopFrameCapture(tabId, "recording"), - ], - { - concurrency: 3, - discard: true, - }, - ); - const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { - const current = tabs.get(tabId); - if (!current) return [Option.none(), tabs] as const; - return [ - Option.some(current), - replaceMap(tabs, (copy) => { - copy.delete(tabId); - }), - ] as const; - }); - if (Option.isNone(tab)) return; - const closedTab = tab.value; - if (closedTab.webContentsId != null) { - yield* Effect.all( - [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], - { concurrency: 2, discard: true }, - ); - } - const updatedAt = yield* currentIso; - const closed: PreviewTabState = { - ...closedTab, - webContentsId: null, - navStatus: { kind: "Idle" }, - canGoBack: false, - canGoForward: false, - zoomFactor: DEFAULT_ZOOM_FACTOR, - pictureInPicture: false, - colorScheme: "system", - controller: "none", - updatedAt, - }; - yield* emit(tabId, closed); - }).pipe( + return yield* withTabLifecycleLock(tabId, closeTabUnlocked(tabId)).pipe( Effect.ensuring( Ref.update(closingTabIdsRef, (closingTabIds) => { if (!closingTabIds.has(tabId)) return closingTabIds; @@ -1438,16 +1458,17 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); - const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { - return yield* withTabLifecycleLock(tabId, closeTabUnlocked(tabId)); - }); - const registerWebviewUnlocked = Effect.fn("PreviewManager.registerWebviewUnlocked")(function* ( tabId: string, webContentsId: number, + expectedGeneration: number | undefined, ) { const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { + if ( + !tab || + tabLifecycleGenerations.get(tabId) !== expectedGeneration || + (yield* Ref.get(closingTabIdsRef)).has(tabId) + ) { return yield* new PreviewTabNotFoundError({ tabId }); } const wc = webContents.fromId(webContentsId); @@ -1501,7 +1522,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const registration = yield* SynchronizedRef.modifyEffect(tabsRef, (tabs) => Effect.gen(function* () { const current = tabs.get(tabId); - if (!current || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { + if ( + !current || + tabLifecycleGenerations.get(tabId) !== expectedGeneration || + (yield* Ref.get(closingTabIdsRef)).has(tabId) + ) { return [ Option.none<{ readonly state: PreviewTabState; readonly pendingUrl: string | null }>(), tabs, @@ -1560,7 +1585,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, webContentsId: number, ) { - return yield* withTabLifecycleLock(tabId, registerWebviewUnlocked(tabId, webContentsId)); + const expectedGeneration = tabLifecycleGenerations.get(tabId); + return yield* withTabLifecycleLock( + tabId, + registerWebviewUnlocked(tabId, webContentsId, expectedGeneration), + ); }); const navigate = Effect.fn("PreviewManager.navigate")(function* (tabId: string, rawUrl: string) { From d43a965cd6a94b898fd64ef4e602cd8a4a91597a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 13:34:13 +0200 Subject: [PATCH 15/26] fix(auth): isolate loopback server sessions Co-authored-by: codex --- .../src/auth/EnvironmentAuthPolicy.test.ts | 5 +- apps/server/src/auth/EnvironmentAuthPolicy.ts | 7 +- apps/server/src/auth/SessionStore.ts | 2 + apps/server/src/auth/utils.test.ts | 66 ++++++++++++++++++- apps/server/src/auth/utils.ts | 33 +++++++++- 5 files changed, 106 insertions(+), 7 deletions(-) diff --git a/apps/server/src/auth/EnvironmentAuthPolicy.test.ts b/apps/server/src/auth/EnvironmentAuthPolicy.test.ts index 95269fb6c37..3227496c432 100644 --- a/apps/server/src/auth/EnvironmentAuthPolicy.test.ts +++ b/apps/server/src/auth/EnvironmentAuthPolicy.test.ts @@ -69,12 +69,13 @@ it.layer(NodeServices.layer)("EnvironmentAuthPolicy.layer", (it) => { expect(descriptor.policy).toBe("loopback-browser"); expect(descriptor.bootstrapMethods).toEqual(["one-time-token"]); - expect(descriptor.sessionCookieName).toBe("t3_session"); + expect(descriptor.sessionCookieName).toMatch(/^t3_session_3773_[a-f0-9]{12}$/); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ mode: "web", host: "127.0.0.1", + port: 3773, }), ), ), @@ -87,6 +88,7 @@ it.layer(NodeServices.layer)("EnvironmentAuthPolicy.layer", (it) => { expect(descriptor.policy).toBe("remote-reachable"); expect(descriptor.bootstrapMethods).toEqual(["one-time-token"]); + expect(descriptor.sessionCookieName).toBe("t3_session"); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ @@ -103,6 +105,7 @@ it.layer(NodeServices.layer)("EnvironmentAuthPolicy.layer", (it) => { const descriptor = yield* policy.getDescriptor(); expect(descriptor.policy).toBe("remote-reachable"); + expect(descriptor.sessionCookieName).toBe("t3_session"); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ diff --git a/apps/server/src/auth/EnvironmentAuthPolicy.ts b/apps/server/src/auth/EnvironmentAuthPolicy.ts index 7ffef0ff0a5..f4a76739fa3 100644 --- a/apps/server/src/auth/EnvironmentAuthPolicy.ts +++ b/apps/server/src/auth/EnvironmentAuthPolicy.ts @@ -4,8 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as ServerConfig from "../config.ts"; -import { resolveSessionCookieName } from "./utils.ts"; -import { isLoopbackHost, isWildcardHost } from "../startupAccess.ts"; +import { isRemoteReachableHost, resolveSessionCookieName } from "./utils.ts"; export class EnvironmentAuthPolicy extends Context.Service< EnvironmentAuthPolicy, @@ -16,7 +15,7 @@ export class EnvironmentAuthPolicy extends Context.Service< export const make = Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - const isRemoteReachable = isWildcardHost(config.host) || !isLoopbackHost(config.host); + const isRemoteReachable = isRemoteReachableHost(config.host); const policy = config.mode === "desktop" @@ -41,6 +40,8 @@ export const make = Effect.gen(function* () { sessionCookieName: resolveSessionCookieName({ mode: config.mode, port: config.port, + host: config.host, + instanceKey: config.stateDir, }), }; diff --git a/apps/server/src/auth/SessionStore.ts b/apps/server/src/auth/SessionStore.ts index 12ecb7dba4d..e079f5c94a9 100644 --- a/apps/server/src/auth/SessionStore.ts +++ b/apps/server/src/auth/SessionStore.ts @@ -470,6 +470,8 @@ export const make = Effect.gen(function* () { const cookieName = resolveSessionCookieName({ mode: serverConfig.mode, port: serverConfig.port, + host: serverConfig.host, + instanceKey: serverConfig.stateDir, }); const emitUpsert = (clientSession: AuthClientSession) => diff --git a/apps/server/src/auth/utils.test.ts b/apps/server/src/auth/utils.test.ts index 90dc0f8ddf9..16e4e2824e9 100644 --- a/apps/server/src/auth/utils.test.ts +++ b/apps/server/src/auth/utils.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vite-plus/test"; -import { deriveAuthClientMetadata } from "./utils.ts"; +import { + deriveAuthClientMetadata, + isRemoteReachableHost, + resolveSessionCookieName, +} from "./utils.ts"; describe("deriveAuthClientMetadata", () => { it("labels Electron user agents as Electron instead of Chrome", () => { @@ -52,3 +56,63 @@ describe("deriveAuthClientMetadata", () => { expect(metadata.userAgent).toContain("Electron/36.3.2"); }); }); + +describe("session cookie isolation", () => { + it("isolates loopback web servers by port and server state", () => { + const first = resolveSessionCookieName({ + mode: "web", + port: 5775, + host: "127.0.0.1", + instanceKey: "/tmp/t3-agent-one", + }); + const second = resolveSessionCookieName({ + mode: "web", + port: 5775, + host: "127.0.0.1", + instanceKey: "/tmp/t3-agent-two", + }); + + expect(first).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + expect(second).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + expect(first).not.toBe(second); + }); + + it("keeps the hosted web cookie stable across server instances", () => { + expect( + resolveSessionCookieName({ + mode: "web", + port: 8080, + host: "0.0.0.0", + instanceKey: "/srv/release-a", + }), + ).toBe("t3_session"); + expect( + resolveSessionCookieName({ + mode: "web", + port: 9090, + host: "app.example.com", + instanceKey: "/srv/release-b", + }), + ).toBe("t3_session"); + }); + + it("retains desktop port scoping", () => { + expect( + resolveSessionCookieName({ + mode: "desktop", + port: 3773, + host: "127.0.0.1", + instanceKey: "/tmp/desktop", + }), + ).toBe("t3_session_3773"); + }); + + it("classifies loopback aliases separately from remotely reachable hosts", () => { + expect(isRemoteReachableHost(undefined)).toBe(false); + expect(isRemoteReachableHost("localhost")).toBe(false); + expect(isRemoteReachableHost("127.12.0.1")).toBe(false); + expect(isRemoteReachableHost("[::1]")).toBe(false); + expect(isRemoteReachableHost("0.0.0.0")).toBe(true); + expect(isRemoteReachableHost("192.168.1.50")).toBe(true); + }); +}); diff --git a/apps/server/src/auth/utils.ts b/apps/server/src/auth/utils.ts index 39f04988ac5..acb66f415fb 100644 --- a/apps/server/src/auth/utils.ts +++ b/apps/server/src/auth/utils.ts @@ -13,12 +13,41 @@ const SESSION_COOKIE_NAME = "t3_session"; export function resolveSessionCookieName(input: { readonly mode: "web" | "desktop"; readonly port: number; + readonly host: string | undefined; + readonly instanceKey: string; }): string { - if (input.mode !== "desktop") { + if (input.mode === "desktop") { + return `${SESSION_COOKIE_NAME}_${input.port}`; + } + + if (isRemoteReachableHost(input.host)) { return SESSION_COOKIE_NAME; } - return `${SESSION_COOKIE_NAME}_${input.port}`; + // Cookies are scoped by host, not port. Loopback development servers need an + // instance-specific name or parallel agents overwrite each other's session, + // and a server that later reuses the port receives a token signed elsewhere. + const instanceHash = NodeCrypto.createHash("sha256") + .update(input.instanceKey) + .digest("hex") + .slice(0, 12); + return `${SESSION_COOKIE_NAME}_${input.port}_${instanceHash}`; +} + +export function isRemoteReachableHost(host: string | undefined): boolean { + if (host === "0.0.0.0" || host === "::" || host === "[::]") { + return true; + } + if (!host || host.length === 0) { + return false; + } + return !( + host === "localhost" || + host === "127.0.0.1" || + host === "::1" || + host === "[::1]" || + host.startsWith("127.") + ); } export function base64UrlEncode(input: string | Uint8Array): string { From 51ac943a020751a32d715457a53f3a28f599871a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 13:38:47 +0200 Subject: [PATCH 16/26] test(auth): use configured session cookie Co-authored-by: codex --- apps/server/src/auth/EnvironmentAuth.test.ts | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/server/src/auth/EnvironmentAuth.test.ts b/apps/server/src/auth/EnvironmentAuth.test.ts index 335e0685197..4227586afc3 100644 --- a/apps/server/src/auth/EnvironmentAuth.test.ts +++ b/apps/server/src/auth/EnvironmentAuth.test.ts @@ -10,6 +10,7 @@ import * as PairingGrantStore from "./PairingGrantStore.ts"; import * as EnvironmentAuth from "./EnvironmentAuth.ts"; import * as ServerSecretStore from "./ServerSecretStore.ts"; +import * as SessionStore from "./SessionStore.ts"; const makeServerConfigLayer = (overrides?: Partial) => Layer.effect( @@ -31,11 +32,12 @@ const makeEnvironmentAuthLayer = (overrides?: Partial[0] => ({ cookies: { - t3_session: sessionToken, + [cookieName]: sessionToken, }, headers: {}, }) as unknown as Parameters< @@ -78,6 +80,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { it.effect("issues standard pairing credentials by default", () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; const pairingCredential = yield* serverAuth.issuePairingCredential(); const exchanged = yield* serverAuth.createBrowserSession( @@ -85,7 +88,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { requestMetadata, ); const verified = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(exchanged.sessionToken), + makeCookieRequest(sessions.cookieName, exchanged.sessionToken), ); expect(verified.sessionId.length).toBeGreaterThan(0); @@ -151,6 +154,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { it.effect("issues startup pairing URLs that bootstrap administrative sessions", () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; const pairingUrl = yield* serverAuth.issueStartupPairingUrl("http://127.0.0.1:3773"); const token = new URLSearchParams(new URL(pairingUrl).hash.slice(1)).get("token"); @@ -164,7 +168,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { const exchanged = yield* serverAuth.createBrowserSession(token ?? "", requestMetadata); const verified = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(exchanged.sessionToken), + makeCookieRequest(sessions.cookieName, exchanged.sessionToken), ); expect(verified.scopes).toEqual([ @@ -186,13 +190,14 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; const administrativeExchange = yield* serverAuth.createBrowserSession( "desktop-bootstrap-token", requestMetadata, ); const administrativeSession = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(administrativeExchange.sessionToken), + makeCookieRequest(sessions.cookieName, administrativeExchange.sessionToken), ); const pairingCredential = yield* serverAuth.issuePairingCredential({ label: "Julius iPhone", @@ -209,7 +214,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { }, ); const clientSession = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(clientExchange.sessionToken), + makeCookieRequest(sessions.cookieName, clientExchange.sessionToken), ); const clientsBeforeRevoke = yield* serverAuth.listClientSessions( administrativeSession.sessionId, From 9bf0e07dbff70b1ebc4829048b07b7cb025a85e3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 13:44:52 +0200 Subject: [PATCH 17/26] fix(auth): isolate wildcard dev sessions Co-authored-by: codex --- .../src/auth/EnvironmentAuthPolicy.test.ts | 19 +++++++++++++++++++ apps/server/src/auth/EnvironmentAuthPolicy.ts | 1 + apps/server/src/auth/SessionStore.ts | 1 + apps/server/src/auth/utils.test.ts | 17 +++++++++++++++++ apps/server/src/auth/utils.ts | 3 ++- 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/apps/server/src/auth/EnvironmentAuthPolicy.test.ts b/apps/server/src/auth/EnvironmentAuthPolicy.test.ts index 3227496c432..1ae29ff0705 100644 --- a/apps/server/src/auth/EnvironmentAuthPolicy.test.ts +++ b/apps/server/src/auth/EnvironmentAuthPolicy.test.ts @@ -99,6 +99,25 @@ it.layer(NodeServices.layer)("EnvironmentAuthPolicy.layer", (it) => { ), ); + it.effect("isolates wildcard-bound web development sessions", () => + Effect.gen(function* () { + const policy = yield* EnvironmentAuthPolicy.EnvironmentAuthPolicy; + const descriptor = yield* policy.getDescriptor(); + + expect(descriptor.policy).toBe("remote-reachable"); + expect(descriptor.sessionCookieName).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + }).pipe( + Effect.provide( + makeEnvironmentAuthPolicyLayer({ + mode: "web", + host: "0.0.0.0", + port: 5775, + devUrl: new URL("http://127.0.0.1:5736"), + }), + ), + ), + ); + it.effect("uses remote-reachable policy for non-loopback web hosts", () => Effect.gen(function* () { const policy = yield* EnvironmentAuthPolicy.EnvironmentAuthPolicy; diff --git a/apps/server/src/auth/EnvironmentAuthPolicy.ts b/apps/server/src/auth/EnvironmentAuthPolicy.ts index f4a76739fa3..9945c69067d 100644 --- a/apps/server/src/auth/EnvironmentAuthPolicy.ts +++ b/apps/server/src/auth/EnvironmentAuthPolicy.ts @@ -42,6 +42,7 @@ export const make = Effect.gen(function* () { port: config.port, host: config.host, instanceKey: config.stateDir, + development: config.devUrl !== undefined, }), }; diff --git a/apps/server/src/auth/SessionStore.ts b/apps/server/src/auth/SessionStore.ts index e079f5c94a9..40a1c43e0be 100644 --- a/apps/server/src/auth/SessionStore.ts +++ b/apps/server/src/auth/SessionStore.ts @@ -472,6 +472,7 @@ export const make = Effect.gen(function* () { port: serverConfig.port, host: serverConfig.host, instanceKey: serverConfig.stateDir, + development: serverConfig.devUrl !== undefined, }); const emitUpsert = (clientSession: AuthClientSession) => diff --git a/apps/server/src/auth/utils.test.ts b/apps/server/src/auth/utils.test.ts index 16e4e2824e9..edc58f71131 100644 --- a/apps/server/src/auth/utils.test.ts +++ b/apps/server/src/auth/utils.test.ts @@ -64,12 +64,14 @@ describe("session cookie isolation", () => { port: 5775, host: "127.0.0.1", instanceKey: "/tmp/t3-agent-one", + development: true, }); const second = resolveSessionCookieName({ mode: "web", port: 5775, host: "127.0.0.1", instanceKey: "/tmp/t3-agent-two", + development: true, }); expect(first).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); @@ -84,6 +86,7 @@ describe("session cookie isolation", () => { port: 8080, host: "0.0.0.0", instanceKey: "/srv/release-a", + development: false, }), ).toBe("t3_session"); expect( @@ -92,6 +95,7 @@ describe("session cookie isolation", () => { port: 9090, host: "app.example.com", instanceKey: "/srv/release-b", + development: false, }), ).toBe("t3_session"); }); @@ -103,10 +107,23 @@ describe("session cookie isolation", () => { port: 3773, host: "127.0.0.1", instanceKey: "/tmp/desktop", + development: true, }), ).toBe("t3_session_3773"); }); + it("isolates development servers even when they bind a wildcard host", () => { + expect( + resolveSessionCookieName({ + mode: "web", + port: 5775, + host: "0.0.0.0", + instanceKey: "/tmp/t3-wildcard-dev", + development: true, + }), + ).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + }); + it("classifies loopback aliases separately from remotely reachable hosts", () => { expect(isRemoteReachableHost(undefined)).toBe(false); expect(isRemoteReachableHost("localhost")).toBe(false); diff --git a/apps/server/src/auth/utils.ts b/apps/server/src/auth/utils.ts index acb66f415fb..b9c53ca8da7 100644 --- a/apps/server/src/auth/utils.ts +++ b/apps/server/src/auth/utils.ts @@ -15,12 +15,13 @@ export function resolveSessionCookieName(input: { readonly port: number; readonly host: string | undefined; readonly instanceKey: string; + readonly development: boolean; }): string { if (input.mode === "desktop") { return `${SESSION_COOKIE_NAME}_${input.port}`; } - if (isRemoteReachableHost(input.host)) { + if (!input.development && isRemoteReachableHost(input.host)) { return SESSION_COOKIE_NAME; } From a9731c11ffacaf5d1089adca1d85f6a8b5f09d1c Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 15:26:06 +0200 Subject: [PATCH 18/26] fix(preview): preserve viewport in resizable PiP Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 6 +- apps/desktop/src/preview/Manager.ts | 3 + apps/web/src/browser/BrowserSurfaceSlot.tsx | 14 ++- apps/web/src/browser/HostedBrowserWebview.tsx | 52 +++++++-- .../src/browser/browserSurfaceStore.test.ts | 31 ++++++ apps/web/src/browser/browserSurfaceStore.ts | 14 ++- .../preview/ThreadPreviewMiniPlayer.tsx | 105 +++++++++++++++++- .../preview/previewMiniPlayerLayout.test.ts | 19 ++++ .../preview/previewMiniPlayerLayout.ts | 36 +++++- apps/web/src/previewMiniPlayerStore.test.ts | 12 ++ apps/web/src/previewMiniPlayerStore.ts | 21 ++++ 11 files changed, 283 insertions(+), 30 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 4dc3675a58a..09be0cd49a2 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -101,7 +101,7 @@ const layer = PreviewManager.layer.pipe( Layer.provideMerge(environmentLayer), Layer.provideMerge(fileSystemLayer), Layer.provideMerge(Path.layer), - Layer.provideMerge(Layer.succeed(HostProcessPlatform, "linux")), + Layer.provideMerge(Layer.succeed(HostProcessPlatform, "darwin")), ); const encodePreviewManagerError = Schema.encodeSync(PreviewManager.PreviewManagerError); @@ -993,6 +993,10 @@ describe("PreviewManager", () => { }), ); expect(pictureInPictureWindow.showInactive).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.setVisibleOnAllWorkspaces).toHaveBeenCalledWith(true, { + visibleOnFullScreen: true, + skipTransformProcessType: true, + }); expect(pictureInPictureWindow.setAspectRatio).toHaveBeenCalledWith(1280 / 720); expect(pictureInPictureSend).toHaveBeenCalledWith( "desktop:preview-pip-frame", diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 28b950ea5c3..7491ebae555 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2299,6 +2299,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function if (hostPlatform === "darwin") { pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true, + // Electron otherwise temporarily transforms the entire app into + // a UIElement process, which removes the owning app from the Dock. + skipTransformProcessType: true, }); } }, diff --git a/apps/web/src/browser/BrowserSurfaceSlot.tsx b/apps/web/src/browser/BrowserSurfaceSlot.tsx index 02b267dbec4..d162c243246 100644 --- a/apps/web/src/browser/BrowserSurfaceSlot.tsx +++ b/apps/web/src/browser/BrowserSurfaceSlot.tsx @@ -10,8 +10,16 @@ export function BrowserSurfaceSlot(props: { readonly cornerRadius?: number; readonly layoutVersion?: string | number; readonly className?: string; + readonly fitSourceContent?: boolean; }) { - const { tabId, visible, cornerRadius = 0, layoutVersion, className } = props; + const { + tabId, + visible, + cornerRadius = 0, + layoutVersion, + className, + fitSourceContent = false, + } = props; const elementRef = useRef(null); const presentationRef = useRef({ visible, cornerRadius }); const updateRef = useRef<(() => void) | null>(null); @@ -19,7 +27,7 @@ export function BrowserSurfaceSlot(props: { useLayoutEffect(() => { const element = elementRef.current; if (!element) return; - const lease = acquireBrowserSurface(tabId); + const lease = acquireBrowserSurface(tabId, fitSourceContent); const update = () => { const rect = element.getBoundingClientRect(); const presentation = presentationRef.current; @@ -47,7 +55,7 @@ export function BrowserSurfaceSlot(props: { if (updateRef.current === update) updateRef.current = null; lease.release(); }; - }, [tabId]); + }, [fitSourceContent, tabId]); useLayoutEffect(() => { presentationRef.current = { visible, cornerRadius }; diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index fe3f5e294ce..ee1086671e8 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -9,7 +9,7 @@ import { usePreviewBridge } from "~/components/preview/usePreviewBridge"; import { cn } from "~/lib/utils"; import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore"; -import { browserViewportSettingKey } from "./browserViewportLayout"; +import { browserViewportSettingKey, resolveBrowserViewportLayout } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles"; import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime"; @@ -57,6 +57,7 @@ export function HostedBrowserWebview(props: { const current = state.byTabId[tabId]; return { cornerRadius: current?.cornerRadius ?? 0, + fittedSourceContent: current?.fittedSourceContent ?? null, rect: resolveBrowserSurfacePanelRect(state.byTabId, tabId), visible: current?.visible ?? false, }; @@ -149,14 +150,15 @@ export function HostedBrowserWebview(props: { } : { width: lastRect?.width ?? 1280, height: lastRect?.height ?? 800 }; const containerSize = active && lastRect ? lastRect : hiddenSize; - const deviceToolbarVisible = active && viewport._tag !== "fill"; + const deviceToolbarVisible = + active && viewport._tag !== "fill" && presentation.fittedSourceContent === null; const { activeDrag, commitViewportChange, effectiveViewport, handleResizeKeyDown, handleResizePointerDown, - layout, + layout: viewportLayout, } = useBrowserViewportResize({ tabId, viewport, @@ -165,6 +167,32 @@ export function HostedBrowserWebview(props: { deviceToolbarVisible, aspectRatio: lockedAspectRatio, }); + const fittedSourceViewport = + presentation.fittedSourceContent && lastRect + ? { + _tag: "freeform" as const, + width: Math.max( + 1, + Math.round( + presentation.fittedSourceContent.width / + presentation.fittedSourceContent.scale / + normalizedZoomFactor, + ), + ), + height: Math.max( + 1, + Math.round( + presentation.fittedSourceContent.height / + presentation.fittedSourceContent.scale / + normalizedZoomFactor, + ), + ), + } + : null; + const layout = + fittedSourceViewport && lastRect + ? resolveBrowserViewportLayout(lastRect, fittedSourceViewport, normalizedZoomFactor) + : viewportLayout; const syncContentPresentation = useCallback(() => { const wrapper = wrapperRef.current; @@ -229,14 +257,18 @@ export function HostedBrowserWebview(props: { data-preview-viewport-mode={effectiveViewport._tag} data-preview-viewport-key={browserViewportSettingKey(effectiveViewport)} data-preview-css-width={ - effectiveViewport._tag === "fill" - ? Math.max(1, Math.round(layout.viewportWidth / normalizedZoomFactor)) - : effectiveViewport.width + fittedSourceViewport + ? fittedSourceViewport.width + : effectiveViewport._tag === "fill" + ? Math.max(1, Math.round(layout.viewportWidth / normalizedZoomFactor)) + : effectiveViewport.width } data-preview-css-height={ - effectiveViewport._tag === "fill" - ? Math.max(1, Math.round(layout.viewportHeight / normalizedZoomFactor)) - : effectiveViewport.height + fittedSourceViewport + ? fittedSourceViewport.height + : effectiveViewport._tag === "fill" + ? Math.max(1, Math.round(layout.viewportHeight / normalizedZoomFactor)) + : effectiveViewport.height } aria-hidden={active ? undefined : true} className={cn( @@ -252,7 +284,7 @@ export function HostedBrowserWebview(props: { transformOrigin: "top left", }} /> - {active && effectiveViewport._tag !== "fill" ? ( + {active && effectiveViewport._tag !== "fill" && !fittedSourceViewport ? ( <> { useBrowserSurfaceStore.setState({ byTabId: {} }); }); + it("freezes the source content dimensions for a fitted presentation", () => { + const tabId = "fitted-browser-surface"; + const sourceOwner = Symbol("source"); + const sourceContent = { + x: 10, + y: 20, + width: 1_280, + height: 720, + scale: 1, + scrollLeft: 0, + scrollTop: 0, + }; + useBrowserSurfaceStore.getState().claim(tabId, sourceOwner, false); + useBrowserSurfaceStore.getState().presentContent(tabId, sourceContent); + + const fittedLease = acquireBrowserSurface(tabId, true); + useBrowserSurfaceStore.getState().presentContent(tabId, { + ...sourceContent, + width: 360, + height: 203, + scale: 0.28125, + }); + + expect(useBrowserSurfaceStore.getState().byTabId[tabId]?.fittedSourceContent).toEqual( + sourceContent, + ); + fittedLease.release(); + }); + it("tracks content dimensions for a browser that has never been visible", () => { const tabId = "hidden-browser-surface-content-test"; useBrowserSurfaceStore.getState().presentContent(tabId, { @@ -40,6 +69,7 @@ describe("browserSurfaceStore", () => { rect: staleRect, visible: false, content: null, + fittedSourceContent: null, cornerRadius: 0, updatedAt: 1, owner: null, @@ -48,6 +78,7 @@ describe("browserSurfaceStore", () => { rect: liveRect, visible: true, content: null, + fittedSourceContent: null, cornerRadius: 0, updatedAt: 2, owner: null, diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index ffa7cf93216..b73638cafac 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -11,6 +11,7 @@ export interface BrowserSurfacePresentation { readonly rect: BrowserSurfaceRect | null; readonly visible: boolean; readonly content: BrowserSurfaceContentPresentation | null; + readonly fittedSourceContent: BrowserSurfaceContentPresentation | null; readonly cornerRadius: number; readonly updatedAt: number; readonly owner: symbol | null; @@ -28,7 +29,7 @@ export interface BrowserSurfaceContentPresentation { interface BrowserSurfaceStoreState { readonly byTabId: Record; - readonly claim: (tabId: string, owner: symbol) => void; + readonly claim: (tabId: string, owner: symbol, fitSourceContent: boolean) => void; readonly present: ( tabId: string, owner: symbol, @@ -74,7 +75,7 @@ const rectEquals = (left: BrowserSurfaceRect | null, right: BrowserSurfaceRect): export const useBrowserSurfaceStore = create()((set) => ({ byTabId: {}, - claim: (tabId, owner) => + claim: (tabId, owner, fitSourceContent) => set((state) => { const current = state.byTabId[tabId]; if (current?.owner === owner) return state; @@ -85,6 +86,7 @@ export const useBrowserSurfaceStore = create()((set) = rect: current?.rect ?? null, visible: false, content: current?.content ?? null, + fittedSourceContent: fitSourceContent ? (current?.content ?? null) : null, cornerRadius: current?.cornerRadius ?? 0, updatedAt: Date.now(), owner, @@ -122,6 +124,7 @@ export const useBrowserSurfaceStore = create()((set) = rect: null, visible: false, content, + fittedSourceContent: null, cornerRadius: 0, updatedAt: Date.now(), owner: null, @@ -162,10 +165,13 @@ export const useBrowserSurfaceStore = create()((set) = }), })); -export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease { +export function acquireBrowserSurface( + tabId: string, + fitSourceContent = false, +): BrowserSurfaceLease { const owner = Symbol(`browser-surface:${tabId}`); let released = false; - useBrowserSurfaceStore.getState().claim(tabId, owner); + useBrowserSurfaceStore.getState().claim(tabId, owner, fitSourceContent); return { present: (rect, visible, cornerRadius = 0) => { diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index 2f81407d8a4..e12dbd1220c 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -12,7 +12,11 @@ import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/prev import { useRightPanelStore } from "~/rightPanelStore"; import { previewBridge } from "./previewBridge"; -import { clampPreviewMiniPlayerPosition } from "./previewMiniPlayerLayout"; +import { + clampPreviewMiniPlayerPosition, + clampPreviewMiniPlayerSize, + PREVIEW_MINI_PLAYER_DEFAULT_SIZE, +} from "./previewMiniPlayerLayout"; interface DragState { readonly pointerId: number; @@ -22,6 +26,14 @@ interface DragState { readonly playerY: number; } +interface ResizeState { + readonly pointerId: number; + readonly pointerX: number; + readonly pointerY: number; + readonly width: number; + readonly height: number; +} + interface Props { readonly threadRef: ScopedThreadRef; readonly tabId: string; @@ -31,6 +43,7 @@ interface Props { export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props) { const rootRef = useRef(null); const dragRef = useRef(null); + const resizeRef = useRef(null); const miniPlayer = usePreviewMiniPlayerStore((state) => selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), ); @@ -38,6 +51,10 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props const snapshot = previewState.sessions[tabId] ?? null; const desktopOverlay = previewState.desktopByTabId[tabId] ?? null; const position = miniPlayer?.tabId === tabId ? miniPlayer.position : null; + const size = + miniPlayer?.tabId === tabId && miniPlayer.size + ? miniPlayer.size + : PREVIEW_MINI_PLAYER_DEFAULT_SIZE; const title = snapshot?.navStatus._tag === "Idle" ? "New tab" @@ -71,10 +88,16 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props const root = rootRef.current; const parent = root?.offsetParent; if (!root || !(parent instanceof HTMLElement)) return; + const nextSize = clampPreviewMiniPlayerSize( + { width: root.offsetWidth, height: root.offsetHeight }, + { width: parent.clientWidth, height: parent.clientHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().resize(threadRef, tabId, nextSize); const next = clampPreviewMiniPlayerPosition( position ?? { x: root.offsetLeft, y: root.offsetTop }, { width: parent.clientWidth, height: parent.clientHeight }, - { width: root.offsetWidth, height: root.offsetHeight }, + nextSize, bottomInset, ); usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); @@ -136,6 +159,60 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props } }; + const handleResizePointerDown = (event: ReactPointerEvent) => { + if (event.button !== 0) return; + const root = rootRef.current; + if (!root) return; + resizeRef.current = { + pointerId: event.pointerId, + pointerX: event.clientX, + pointerY: event.clientY, + width: root.offsetWidth, + height: root.offsetHeight, + }; + event.currentTarget.setPointerCapture(event.pointerId); + event.preventDefault(); + event.stopPropagation(); + }; + + const handleResizePointerMove = (event: ReactPointerEvent) => { + const resize = resizeRef.current; + const root = rootRef.current; + const parent = root?.offsetParent; + if ( + !resize || + resize.pointerId !== event.pointerId || + !root || + !(parent instanceof HTMLElement) + ) { + return; + } + const nextSize = clampPreviewMiniPlayerSize( + { + width: resize.width + event.clientX - resize.pointerX, + height: resize.height + event.clientY - resize.pointerY, + }, + { width: parent.clientWidth, height: parent.clientHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().resize(threadRef, tabId, nextSize); + const nextPosition = clampPreviewMiniPlayerPosition( + position ?? { x: root.offsetLeft, y: root.offsetTop }, + { width: parent.clientWidth, height: parent.clientHeight }, + nextSize, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, nextPosition); + }; + + const endResize = (event: ReactPointerEvent) => { + if (resizeRef.current?.pointerId !== event.pointerId) return; + resizeRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + }; + if (!snapshot || miniPlayer?.tabId !== tabId) return null; return ( @@ -143,11 +220,16 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props ref={rootRef} aria-label="Floating browser preview" data-preview-mini-player={tabId} - className="pointer-events-none absolute w-[360px] max-w-[calc(100%-24px)] select-none" + className="pointer-events-none absolute select-none" style={ position - ? { left: position.x, top: position.y } - : { right: 16, bottom: Math.max(16, bottomInset + 16) } + ? { left: position.x, top: position.y, width: size.width, height: size.height } + : { + right: 16, + bottom: Math.max(16, bottomInset + 16), + width: size.width, + height: size.height, + } } >
-
+
@@ -223,6 +306,16 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props Reconnecting preview…
) : null} +
); diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts index 762118fabc9..84ae930880c 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test"; import { clampPreviewMiniPlayerPosition, + clampPreviewMiniPlayerSize, PREVIEW_MINI_PLAYER_EDGE_GAP, } from "./previewMiniPlayerLayout"; @@ -46,3 +47,21 @@ describe("clampPreviewMiniPlayerPosition", () => { }); }); }); + +describe("clampPreviewMiniPlayerSize", () => { + it("allows resizing within the available chat viewport", () => { + expect( + clampPreviewMiniPlayerSize({ width: 520, height: 360 }, { width: 1_000, height: 700 }, 120), + ).toEqual({ width: 520, height: 360 }); + }); + + it("bounds oversized players above the composer", () => { + expect( + clampPreviewMiniPlayerSize( + { width: 2_000, height: 2_000 }, + { width: 1_000, height: 700 }, + 120, + ), + ).toEqual({ width: 976, height: 556 }); + }); +}); diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index d1a04c3143e..0382a6d2228 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -1,11 +1,35 @@ -import type { PreviewMiniPlayerPosition } from "~/previewMiniPlayerStore"; - -export interface PreviewMiniPlayerSize { - readonly width: number; - readonly height: number; -} +import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; +export const PREVIEW_MINI_PLAYER_DEFAULT_SIZE = { width: 360, height: 239 } as const; +export const PREVIEW_MINI_PLAYER_MIN_SIZE = { width: 280, height: 194 } as const; + +export function clampPreviewMiniPlayerSize( + size: PreviewMiniPlayerSize, + container: PreviewMiniPlayerSize, + bottomInset = 0, +): PreviewMiniPlayerSize { + return { + width: Math.round( + Math.min( + Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.width, size.width), + Math.max( + PREVIEW_MINI_PLAYER_MIN_SIZE.width, + container.width - PREVIEW_MINI_PLAYER_EDGE_GAP * 2, + ), + ), + ), + height: Math.round( + Math.min( + Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.height, size.height), + Math.max( + PREVIEW_MINI_PLAYER_MIN_SIZE.height, + container.height - Math.max(0, bottomInset) - PREVIEW_MINI_PLAYER_EDGE_GAP * 2, + ), + ), + ), + }; +} export function clampPreviewMiniPlayerPosition( position: PreviewMiniPlayerPosition, diff --git a/apps/web/src/previewMiniPlayerStore.test.ts b/apps/web/src/previewMiniPlayerStore.test.ts index d0d0e7a8a5e..d6ec64bf069 100644 --- a/apps/web/src/previewMiniPlayerStore.test.ts +++ b/apps/web/src/previewMiniPlayerStore.test.ts @@ -34,6 +34,7 @@ describe("previewMiniPlayerStore", () => { ).toEqual({ tabId: "tab-b", position: { x: 24, y: 48 }, + size: null, }); }); @@ -47,6 +48,17 @@ describe("previewMiniPlayerStore", () => { ).toEqual({ tabId: "tab-b", position: null, + size: null, }); }); + + it("preserves a thread-bound size while switching tabs", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().resize(refA, "tab-a", { width: 480, height: 320 }); + usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toMatchObject({ tabId: "tab-b", size: { width: 480, height: 320 } }); + }); }); diff --git a/apps/web/src/previewMiniPlayerStore.ts b/apps/web/src/previewMiniPlayerStore.ts index 5001c096bb4..d1a1fde5eff 100644 --- a/apps/web/src/previewMiniPlayerStore.ts +++ b/apps/web/src/previewMiniPlayerStore.ts @@ -7,9 +7,15 @@ export interface PreviewMiniPlayerPosition { readonly y: number; } +export interface PreviewMiniPlayerSize { + readonly width: number; + readonly height: number; +} + export interface PreviewMiniPlayerState { readonly tabId: string; readonly position: PreviewMiniPlayerPosition | null; + readonly size: PreviewMiniPlayerSize | null; } interface PreviewMiniPlayerStoreState { @@ -17,6 +23,7 @@ interface PreviewMiniPlayerStoreState { readonly open: (ref: ScopedThreadRef, tabId: string) => void; readonly close: (ref: ScopedThreadRef) => void; readonly move: (ref: ScopedThreadRef, tabId: string, position: PreviewMiniPlayerPosition) => void; + readonly resize: (ref: ScopedThreadRef, tabId: string, size: PreviewMiniPlayerSize) => void; readonly removeThread: (ref: ScopedThreadRef) => void; } @@ -33,6 +40,7 @@ export const usePreviewMiniPlayerStore = create()(( [threadKey]: { tabId, position: current?.position ?? null, + size: current?.size ?? null, }, }, }; @@ -57,6 +65,19 @@ export const usePreviewMiniPlayerStore = create()(( }, }; }), + resize: (ref, tabId, size) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const current = state.byThreadKey[threadKey]; + if (!current || current.tabId !== tabId) return state; + if (current.size?.width === size.width && current.size.height === size.height) return state; + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { ...current, size }, + }, + }; + }), removeThread: (ref) => set((state) => { const threadKey = scopedThreadKey(ref); From 3ff68d56b4c56ea908e8686917fac27977dc0ca3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 15:50:23 +0200 Subject: [PATCH 19/26] fix(preview): harden background lifecycle Co-authored-by: codex --- apps/desktop/src/preview/Manager.ts | 12 +++++- apps/web/src/browser/BrowserSurfaceSlot.tsx | 2 +- apps/web/src/browser/HostedBrowserWebview.tsx | 15 +++++-- apps/web/src/browser/browserRecording.test.ts | 40 ++++++++++--------- apps/web/src/browser/browserRecording.ts | 9 +++-- .../preview/previewMiniPlayerLayout.test.ts | 6 +++ .../preview/previewMiniPlayerLayout.ts | 21 ++++------ 7 files changed, 64 insertions(+), 41 deletions(-) diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 7491ebae555..bd537688760 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -1505,13 +1505,21 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { concurrency: 3, discard: true }, ); } + const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if ( + !currentTab || + tabLifecycleGenerations.get(tabId) !== expectedGeneration || + (yield* Ref.get(closingTabIdsRef)).has(tabId) + ) { + return yield* new PreviewTabNotFoundError({ tabId }); + } const zoomFactor = replacedWebContentsId !== null ? yield* attempt( { operation: "registerWebview.restoreZoomFactor", tabId, webContentsId }, () => { - wc.setZoomFactor(tab.zoomFactor); - return tab.zoomFactor; + wc.setZoomFactor(currentTab.zoomFactor); + return currentTab.zoomFactor; }, ) : yield* attempt({ operation: "registerWebview.getZoomFactor", tabId, webContentsId }, () => diff --git a/apps/web/src/browser/BrowserSurfaceSlot.tsx b/apps/web/src/browser/BrowserSurfaceSlot.tsx index d162c243246..6b22bdd318a 100644 --- a/apps/web/src/browser/BrowserSurfaceSlot.tsx +++ b/apps/web/src/browser/BrowserSurfaceSlot.tsx @@ -55,7 +55,7 @@ export function BrowserSurfaceSlot(props: { if (updateRef.current === update) updateRef.current = null; lease.release(); }; - }, [fitSourceContent, tabId]); + }, [fitSourceContent, tabId, visible]); useLayoutEffect(() => { presentationRef.current = { visible, cornerRadius }; diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index ee1086671e8..831fe90d629 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -76,6 +76,12 @@ export function HostedBrowserWebview(props: { }, [tabId]); const [webviewGeneration, setWebviewGeneration] = useState(0); + const [recoverySrc, setRecoverySrc] = useState(initialSrc); + const latestUrlRef = useRef(initialUrl); + + useEffect(() => { + latestUrlRef.current = initialUrl; + }, [initialUrl]); const setWebviewRef = useCallback((node: HTMLElement | null) => { webviewRef.current = node as ElectronWebview | null; @@ -114,7 +120,10 @@ export function HostedBrowserWebview(props: { crashRecoveryRef.current = recovery.state; recoveryTimeout = setTimeout(() => { recoveryTimeout = null; - if (!disposed) setWebviewGeneration((generation) => generation + 1); + if (!disposed) { + setRecoverySrc(latestUrlRef.current ?? initialSrc); + setWebviewGeneration((generation) => generation + 1); + } }, recovery.delayMs); }; webview.addEventListener("did-attach", register); @@ -128,7 +137,7 @@ export function HostedBrowserWebview(props: { webview.removeEventListener("dom-ready", register); webview.removeEventListener("render-process-gone", recoverGuest); }; - }, [config, tabId, webviewGeneration]); + }, [config, initialSrc, tabId, webviewGeneration]); const active = presentation.visible && presentation.rect !== null; const lastRect = presentation.rect; @@ -249,7 +258,7 @@ export function HostedBrowserWebview(props: { { + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "startup-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:00.000Z", + }); +}; + describe("browser recording", () => { beforeEach(() => { events.length = 0; @@ -425,28 +434,27 @@ describe("browser recording", () => { expect(save).toHaveBeenCalledOnce(); }); - it("stops a screencast that finishes starting after cancellation", async () => { + it("finishes startup before stopping so an active recording yields an artifact", async () => { let finishStartingScreencast: (() => void) | undefined; startScreencast.mockImplementationOnce(async () => { events.push("start-screencast"); await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); const startPromise = startBrowserRecording("recording-tab"); - const rejectedStart = expect(startPromise).rejects.toBeInstanceOf( - BrowserRecordingOperationError, - ); await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); const stopPromise = stopBrowserRecording("recording-tab"); await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce()); finishStartingScreencast?.(); - await rejectedStart; - await stopPromise; - expect(stopScreencast).toHaveBeenCalledTimes(2); + await startPromise; + await expect(stopPromise).resolves.toMatchObject({ tabId: "recording-tab" }); + expect(stopScreencast).toHaveBeenCalledOnce(); + expect(save).toHaveBeenCalledOnce(); expect(events.at(-1)).toBe("clear"); }); @@ -457,12 +465,10 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); const firstStart = startBrowserRecording("recording-tab"); - const rejectedFirstStart = expect(firstStart).rejects.toBeInstanceOf( - BrowserRecordingOperationError, - ); await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); const stopPromise = stopBrowserRecording("recording-tab"); @@ -471,7 +477,7 @@ describe("browser recording", () => { const startCallsBeforeFirstSettled = startScreencast.mock.calls.length; finishStartingScreencast?.(); - await rejectedFirstStart; + await firstStart; await stopPromise; await restartAfterStop; await stopBrowserRecording("recording-tab"); @@ -486,11 +492,11 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); stopScreencast.mockRejectedValueOnce(new Error("initial stop failed")); const firstStart = startBrowserRecording("recording-tab"); - const rejectedStart = expect(firstStart).rejects.toBeInstanceOf(BrowserRecordingOperationError); await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); const stopPromise = stopBrowserRecording("recording-tab"); @@ -504,7 +510,7 @@ describe("browser recording", () => { ); finishStartingScreencast?.(); - await rejectedStart; + await firstStart; await rejectedStop; await startBrowserRecording("recording-tab"); @@ -519,12 +525,10 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); const startPromise = startBrowserRecording("recording-tab"); - const rejectedStart = expect(startPromise).rejects.toBeInstanceOf( - BrowserRecordingOperationError, - ); expect(startScreencast).toHaveBeenCalledOnce(); const stopPromise = stopBrowserRecording("recording-tab"); @@ -545,7 +549,7 @@ describe("browser recording", () => { ); finishStartingScreencast?.(); - await rejectedStart; + await startPromise; const cleanupResult = await stopBrowserRecording("recording-tab"); expect(cleanupResult).toBeNull(); expect(save).not.toHaveBeenCalled(); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 56211e95351..fa247457868 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -362,7 +362,9 @@ export async function startBrowserRecording( }); } const throwIfStartupCancelled = async (): Promise => { - if (isRecordingStarting(recording)) return; + // A stop requested during startup should let startup finish so the + // caller receives a real artifact. Only replacement/removal cancels it. + if (activeRecordings.get(tabId) === recording) return; try { await bridge.recording.stopScreencast(tabId); } catch (cause) { @@ -442,7 +444,9 @@ export async function startBrowserRecording( ), }); } - recording.lifecycle = { phase: "recording" }; + if (recording.lifecycle.phase === "starting") { + recording.lifecycle = { phase: "recording" }; + } appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { tabIds: new Set(activeRecordings.keys()), }); @@ -583,7 +587,6 @@ export function stopBrowserRecording( if (!bridge || !recording) return Promise.resolve(null); if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; - recording.settleFirstFrameSize("cancelled"); const stopPromise = Promise.resolve() .then(() => finalizeBrowserRecording(bridge, recording)) .catch((error) => { diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts index 84ae930880c..98b26e8a0b2 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts @@ -64,4 +64,10 @@ describe("clampPreviewMiniPlayerSize", () => { ), ).toEqual({ width: 976, height: 556 }); }); + + it("lets a tiny container win over the preferred minimum", () => { + expect( + clampPreviewMiniPlayerSize({ width: 360, height: 239 }, { width: 250, height: 180 }, 20), + ).toEqual({ width: 226, height: 136 }); + }); }); diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index 0382a6d2228..d0388d71d21 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -9,24 +9,17 @@ export function clampPreviewMiniPlayerSize( container: PreviewMiniPlayerSize, bottomInset = 0, ): PreviewMiniPlayerSize { + const availableWidth = Math.max(1, container.width - PREVIEW_MINI_PLAYER_EDGE_GAP * 2); + const availableHeight = Math.max( + 1, + container.height - Math.max(0, bottomInset) - PREVIEW_MINI_PLAYER_EDGE_GAP * 2, + ); return { width: Math.round( - Math.min( - Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.width, size.width), - Math.max( - PREVIEW_MINI_PLAYER_MIN_SIZE.width, - container.width - PREVIEW_MINI_PLAYER_EDGE_GAP * 2, - ), - ), + Math.min(Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.width, size.width), availableWidth), ), height: Math.round( - Math.min( - Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.height, size.height), - Math.max( - PREVIEW_MINI_PLAYER_MIN_SIZE.height, - container.height - Math.max(0, bottomInset) - PREVIEW_MINI_PLAYER_EDGE_GAP * 2, - ), - ), + Math.min(Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.height, size.height), availableHeight), ), }; } From 83703e7480f8f4e847a465e0439dcda1880f97de Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 15:58:58 +0200 Subject: [PATCH 20/26] fix(preview): close remaining capture races Co-authored-by: codex --- apps/web/src/browser/BrowserSurfaceSlot.tsx | 20 +++++++++++++++--- apps/web/src/browser/browserRecording.test.ts | 8 ++++--- apps/web/src/browser/browserRecording.ts | 21 ++----------------- .../src/browser/browserSurfaceStore.test.ts | 2 +- apps/web/src/browser/browserSurfaceStore.ts | 6 ++++-- 5 files changed, 29 insertions(+), 28 deletions(-) diff --git a/apps/web/src/browser/BrowserSurfaceSlot.tsx b/apps/web/src/browser/BrowserSurfaceSlot.tsx index 6b22bdd318a..a9d3f541ff1 100644 --- a/apps/web/src/browser/BrowserSurfaceSlot.tsx +++ b/apps/web/src/browser/BrowserSurfaceSlot.tsx @@ -27,11 +27,11 @@ export function BrowserSurfaceSlot(props: { useLayoutEffect(() => { const element = elementRef.current; if (!element) return; - const lease = acquireBrowserSurface(tabId, fitSourceContent); + let lease = acquireBrowserSurface(tabId, fitSourceContent); const update = () => { const rect = element.getBoundingClientRect(); const presentation = presentationRef.current; - lease.present( + const presented = lease.present( { x: Math.round(rect.x), y: Math.round(rect.y), @@ -41,6 +41,20 @@ export function BrowserSurfaceSlot(props: { presentation.visible && rect.width > 0 && rect.height > 0, presentation.cornerRadius, ); + if (presentation.visible && !presented) { + lease.release(); + lease = acquireBrowserSurface(tabId, fitSourceContent); + lease.present( + { + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.max(1, Math.round(rect.width)), + height: Math.max(1, Math.round(rect.height)), + }, + rect.width > 0 && rect.height > 0, + presentation.cornerRadius, + ); + } }; updateRef.current = update; update(); @@ -55,7 +69,7 @@ export function BrowserSurfaceSlot(props: { if (updateRef.current === update) updateRef.current = null; lease.release(); }; - }, [fitSourceContent, tabId, visible]); + }, [fitSourceContent, tabId]); useLayoutEffect(() => { presentationRef.current = { visible, cornerRadius }; diff --git a/apps/web/src/browser/browserRecording.test.ts b/apps/web/src/browser/browserRecording.test.ts index a2a47a50142..c7208055da3 100644 --- a/apps/web/src/browser/browserRecording.test.ts +++ b/apps/web/src/browser/browserRecording.test.ts @@ -448,7 +448,7 @@ describe("browser recording", () => { await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); const stopPromise = stopBrowserRecording("recording-tab"); - await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce()); + expect(stopScreencast).not.toHaveBeenCalled(); finishStartingScreencast?.(); await startPromise; @@ -504,7 +504,7 @@ describe("browser recording", () => { operation: "stop-screencast", tabId: "recording-tab", }); - await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce()); + expect(stopScreencast).not.toHaveBeenCalled(); await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf( BrowserRecordingConflictError, ); @@ -512,6 +512,7 @@ describe("browser recording", () => { finishStartingScreencast?.(); await firstStart; await rejectedStop; + expect(stopScreencast).toHaveBeenCalledOnce(); await startBrowserRecording("recording-tab"); await stopBrowserRecording("recording-tab"); @@ -534,7 +535,7 @@ describe("browser recording", () => { const stopPromise = stopBrowserRecording("recording-tab"); await Promise.resolve(); await Promise.resolve(); - expect(stopScreencast).toHaveBeenCalledOnce(); + expect(stopScreencast).not.toHaveBeenCalled(); const rejection = expect(stopPromise).rejects.toMatchObject({ operation: "wait-startup", @@ -552,6 +553,7 @@ describe("browser recording", () => { await startPromise; const cleanupResult = await stopBrowserRecording("recording-tab"); expect(cleanupResult).toBeNull(); + expect(stopScreencast).toHaveBeenCalledOnce(); expect(save).not.toHaveBeenCalled(); expect(events.at(-1)).toBe("clear"); }); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index fa247457868..5b88dfd4133 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -468,33 +468,16 @@ const finalizeBrowserRecording = async ( } | { readonly _tag: "Failure"; readonly error: unknown }; try { - let stopScreencastError: BrowserRecordingOperationError | undefined; + await waitForRecordingStartupToSettle(recording); try { await bridge.recording.stopScreencast(tabId); } catch (cause) { - stopScreencastError = new BrowserRecordingOperationError({ + throw new BrowserRecordingOperationError({ operation: "stop-screencast", tabId, cause, }); } - try { - await waitForRecordingStartupToSettle(recording); - } catch (startupError) { - if (stopScreencastError) { - throw new BrowserRecordingOperationError({ - operation: "wait-startup", - tabId, - cause: new AggregateError( - [startupError, stopScreencastError], - `Browser recording stop failed while startup remained pending for tab ${tabId}.`, - { cause: startupError }, - ), - }); - } - throw startupError; - } - if (stopScreencastError) throw stopScreencastError; if (!recording.recorder || !recording.mimeType) { result = { _tag: "Success", artifact: null }; } else { diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index 2bd4315bb04..dcc3f8ebd69 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -98,7 +98,7 @@ describe("browserSurfaceStore", () => { const liveLease = acquireBrowserSurface(tabId); liveLease.present(liveRect, true); - staleLease.present(staleRect, true); + expect(staleLease.present(staleRect, true)).toBe(false); staleLease.release(); expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({ diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index b73638cafac..ff7425023b6 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -42,7 +42,7 @@ interface BrowserSurfaceStoreState { } export interface BrowserSurfaceLease { - readonly present: (rect: BrowserSurfaceRect, visible: boolean, cornerRadius?: number) => void; + readonly present: (rect: BrowserSurfaceRect, visible: boolean, cornerRadius?: number) => boolean; readonly release: () => void; } @@ -175,8 +175,10 @@ export function acquireBrowserSurface( return { present: (rect, visible, cornerRadius = 0) => { - if (released) return; + if (released) return false; + if (useBrowserSurfaceStore.getState().byTabId[tabId]?.owner !== owner) return false; useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible, cornerRadius); + return true; }, release: () => { if (released) return; From 3435b33c01058ec208b5a308eb09732250ff1e15 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 16:24:34 +0200 Subject: [PATCH 21/26] fix(preview): compact and scale PiP surfaces Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 12 ++ apps/desktop/src/preview/Manager.ts | 28 ++++- .../preview/ThreadPreviewMiniPlayer.tsx | 103 ++++++++---------- .../preview/previewMiniPlayerLayout.ts | 4 +- 4 files changed, 84 insertions(+), 63 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 09be0cd49a2..c0f23a52f6e 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -21,6 +21,13 @@ import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as BrowserSession from "./BrowserSession.ts"; import * as PreviewManager from "./Manager.ts"; +describe("fitPictureInPictureContentSize", () => { + it("fits landscape and portrait viewports without letterboxing the window", () => { + expect(PreviewManager.fitPictureInPictureContentSize([480, 320], 16 / 9)).toEqual([480, 270]); + expect(PreviewManager.fitPictureInPictureContentSize([480, 320], 9 / 16)).toEqual([240, 427]); + }); +}); + const { browserWindowConstructor, createFromPath, @@ -161,6 +168,8 @@ const makeTestPictureInPictureWindow = (loadURL: () => Promise = async () setAlwaysOnTop: vi.fn(), setVisibleOnAllWorkspaces: vi.fn(), setAspectRatio: vi.fn(), + getContentSize: vi.fn(() => [480, 320]), + setContentSize: vi.fn(), loadURL: vi.fn(loadURL), showInactive: vi.fn(() => { if (destroyed) throw new Error("Picture-in-picture window is closed."); @@ -952,6 +961,8 @@ describe("PreviewManager", () => { setAlwaysOnTop: vi.fn(), setVisibleOnAllWorkspaces: vi.fn(), setAspectRatio: vi.fn(), + getContentSize: vi.fn(() => [480, 320] as [number, number]), + setContentSize: vi.fn(), loadURL: vi.fn(async () => undefined), showInactive: vi.fn(), close: vi.fn(() => { @@ -998,6 +1009,7 @@ describe("PreviewManager", () => { skipTransformProcessType: true, }); expect(pictureInPictureWindow.setAspectRatio).toHaveBeenCalledWith(1280 / 720); + expect(pictureInPictureWindow.setContentSize).toHaveBeenCalledWith(480, 270, false); expect(pictureInPictureSend).toHaveBeenCalledWith( "desktop:preview-pip-frame", expect.objectContaining({ diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index bd537688760..01a9b7de517 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -158,6 +158,25 @@ export const buildPreviewPictureInPictureDataUrl = (): string => { return `data:text/html;charset=utf-8,${encodeURIComponent(html)}`; }; +export const fitPictureInPictureContentSize = ( + current: ReadonlyArray, + aspectRatio: number, +): readonly [width: number, height: number] => { + const currentWidth = Math.max(1, current[0] ?? PICTURE_IN_PICTURE_INITIAL_WIDTH); + const currentHeight = Math.max(1, current[1] ?? PICTURE_IN_PICTURE_INITIAL_HEIGHT); + let width = + currentWidth / currentHeight > aspectRatio ? currentHeight * aspectRatio : currentWidth; + let height = width / aspectRatio; + const minimumScale = Math.max( + 1, + PICTURE_IN_PICTURE_MIN_WIDTH / width, + PICTURE_IN_PICTURE_MIN_HEIGHT / height, + ); + width *= minimumScale; + height *= minimumScale; + return [Math.round(width), Math.round(height)]; +}; + const artifactSiteSlug = (rawUrl: string): string => { try { const url = new URL(rawUrl); @@ -2060,7 +2079,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId, webContentsId: wc.id, }, - () => pictureInPictureWindow.setAspectRatio(aspectRatio), + () => { + const contentSize = fitPictureInPictureContentSize( + pictureInPictureWindow.getContentSize(), + aspectRatio, + ); + pictureInPictureWindow.setContentSize(contentSize[0], contentSize[1], false); + pictureInPictureWindow.setAspectRatio(aspectRatio); + }, ); yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => replaceMap(aspectRatios, (copy) => { diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index e12dbd1220c..dd94306c6e8 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -55,11 +55,6 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props miniPlayer?.tabId === tabId && miniPlayer.size ? miniPlayer.size : PREVIEW_MINI_PLAYER_DEFAULT_SIZE; - const title = - snapshot?.navStatus._tag === "Idle" - ? "New tab" - : snapshot?.navStatus.title || snapshot?.navStatus.url || "Preview"; - const close = () => { usePreviewMiniPlayerStore.getState().close(threadRef); }; @@ -226,72 +221,60 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props ? { left: position.x, top: position.y, width: size.width, height: size.height } : { right: 16, - bottom: Math.max(16, bottomInset + 16), + top: 16, width: size.width, height: size.height, } } >
-
- - - -
- - Preview · {title} - -
- - - -
+ + +
-
-
+
+
-
+
{!desktopOverlay?.hasWebContents ? ( -
+
Reconnecting preview…
) : null} diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts index d0388d71d21..abdd5b8ca4d 100644 --- a/apps/web/src/components/preview/previewMiniPlayerLayout.ts +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -1,8 +1,8 @@ import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; -export const PREVIEW_MINI_PLAYER_DEFAULT_SIZE = { width: 360, height: 239 } as const; -export const PREVIEW_MINI_PLAYER_MIN_SIZE = { width: 280, height: 194 } as const; +export const PREVIEW_MINI_PLAYER_DEFAULT_SIZE = { width: 320, height: 200 } as const; +export const PREVIEW_MINI_PLAYER_MIN_SIZE = { width: 240, height: 150 } as const; export function clampPreviewMiniPlayerSize( size: PreviewMiniPlayerSize, From 75f8378c859fe5999e008cc7119805c36347460f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 16:29:58 +0200 Subject: [PATCH 22/26] fix(preview): reset native PiP aspect lock Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 10 ++++++++-- apps/desktop/src/preview/Manager.ts | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index c0f23a52f6e..94fdd45550e 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1008,8 +1008,14 @@ describe("PreviewManager", () => { visibleOnFullScreen: true, skipTransformProcessType: true, }); - expect(pictureInPictureWindow.setAspectRatio).toHaveBeenCalledWith(1280 / 720); + expect(pictureInPictureWindow.setAspectRatio.mock.calls).toEqual([[0], [1280 / 720]]); expect(pictureInPictureWindow.setContentSize).toHaveBeenCalledWith(480, 270, false); + expect(pictureInPictureWindow.setAspectRatio.mock.invocationCallOrder[0]).toBeLessThan( + pictureInPictureWindow.setContentSize.mock.invocationCallOrder[0] ?? 0, + ); + expect(pictureInPictureWindow.setContentSize.mock.invocationCallOrder[0]).toBeLessThan( + pictureInPictureWindow.setAspectRatio.mock.invocationCallOrder[1] ?? 0, + ); expect(pictureInPictureSend).toHaveBeenCalledWith( "desktop:preview-pip-frame", expect.objectContaining({ @@ -1119,7 +1125,7 @@ describe("PreviewManager", () => { yield* TestClock.adjust(100); - expect(pictureInPictureWindow.setAspectRatio).toHaveBeenCalledWith(1280 / 720); + expect(pictureInPictureWindow.setAspectRatio.mock.calls).toEqual([[0], [1280 / 720]]); expect(send).toHaveBeenCalledOnce(); yield* manager.closePictureInPicture("tab_empty_frame"); }), diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 01a9b7de517..f4b10157079 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2084,6 +2084,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function pictureInPictureWindow.getContentSize(), aspectRatio, ); + pictureInPictureWindow.setAspectRatio(0); pictureInPictureWindow.setContentSize(contentSize[0], contentSize[1], false); pictureInPictureWindow.setAspectRatio(aspectRatio); }, From 3f818ccceb1b0a85e86e80aa010e8a37f0c3effe Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 16:48:16 +0200 Subject: [PATCH 23/26] fix(preview): open inline player by default Co-authored-by: codex --- .../src/mcp/toolkits/preview/handlers.test.ts | 32 +++++++++++++++++++ .../src/mcp/toolkits/preview/handlers.ts | 17 +++++++--- apps/server/src/mcp/toolkits/preview/tools.ts | 2 +- .../preview/PreviewAutomationHosts.tsx | 11 ++++--- .../preview/ThreadPreviewMiniPlayer.tsx | 2 +- .../previewAutomationOpenReadiness.test.ts | 17 +++++++++- .../preview/previewAutomationOpenReadiness.ts | 4 +++ packages/contracts/src/preview.test.ts | 10 ++++++ packages/contracts/src/previewAutomation.ts | 9 +++++- 9 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 apps/server/src/mcp/toolkits/preview/handlers.test.ts diff --git a/apps/server/src/mcp/toolkits/preview/handlers.test.ts b/apps/server/src/mcp/toolkits/preview/handlers.test.ts new file mode 100644 index 00000000000..4cfdb0c39b6 --- /dev/null +++ b/apps/server/src/mcp/toolkits/preview/handlers.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { normalizePreviewOpenInput } from "./handlers.ts"; + +describe("normalizePreviewOpenInput", () => { + it("opens the inline preview and reuses the current tab by default", () => { + expect(normalizePreviewOpenInput({})).toEqual({ + open: true, + reuseExistingTab: true, + }); + }); + + it("preserves an explicit background-only opt-out", () => { + expect(normalizePreviewOpenInput({ open: false })).toEqual({ + open: false, + reuseExistingTab: true, + }); + }); + + it("supports show as a legacy alias while preferring open", () => { + expect(normalizePreviewOpenInput({ show: false })).toEqual({ + open: false, + reuseExistingTab: true, + show: false, + }); + expect(normalizePreviewOpenInput({ open: true, show: false })).toEqual({ + open: true, + reuseExistingTab: true, + show: false, + }); + }); +}); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 8c4651dc1cf..484784b911a 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -1,6 +1,7 @@ import * as Effect from "effect/Effect"; import type { PreviewAutomationOperation, + PreviewAutomationOpenInput, PreviewAutomationRecordingArtifact, PreviewAutomationRecordingStatus, PreviewAutomationResizeResult, @@ -14,6 +15,16 @@ import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts"; import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts"; +export function normalizePreviewOpenInput( + input: PreviewAutomationOpenInput, +): PreviewAutomationOpenInput { + return { + ...input, + open: input.open ?? input.show ?? true, + reuseExistingTab: input.reuseExistingTab ?? true, + }; +} + const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( operation: PreviewAutomationOperation, input: unknown, @@ -50,11 +61,7 @@ const invokeTargeted = ( const handlers = { preview_status: (input) => invokeTargeted("status", input ?? {}), preview_open: (input) => - invokeTargeted("open", { - ...input, - show: input.show ?? true, - reuseExistingTab: input.reuseExistingTab ?? true, - }), + invokeTargeted("open", normalizePreviewOpenInput(input)), preview_navigate: (input) => invokeTargeted("navigate", input, input.timeoutMs), preview_resize: (input) => diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 74e569252dd..a94d2b056f7 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -54,7 +54,7 @@ export const PreviewStatusTool = Tool.make("preview_status", { export const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", + "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewAutomationError, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index e38b91227a6..165fb6136aa 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -29,7 +29,7 @@ import { reconcilePreviewServerSessions, updatePreviewServerSnapshot, } from "~/previewStateStore"; -import { useRightPanelStore } from "~/rightPanelStore"; +import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; import { readActiveBrowserRecordingTabIds, @@ -53,7 +53,10 @@ import { PreviewAutomationTargetUnavailableError, PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; -import { previewAutomationOpenNeedsOverlay } from "./previewAutomationOpenReadiness"; +import { + previewAutomationOpenNeedsOverlay, + shouldOpenPreviewMiniPlayer, +} from "./previewAutomationOpenReadiness"; import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { @@ -378,8 +381,8 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) activeSnapshot = snapshot; tabId = activeTabId; } - if (input.show ?? true) { - useRightPanelStore.getState().openBrowser(threadRef, activeTabId); + if (shouldOpenPreviewMiniPlayer(input)) { + usePreviewMiniPlayerStore.getState().open(threadRef, activeTabId); } if (activeSnapshot && previewAutomationOpenNeedsOverlay(input, activeSnapshot)) { await waitForDesktopOverlay( diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index dd94306c6e8..b6e5b372d07 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -228,7 +228,7 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props } >
({ threadId: "thread-1", @@ -13,6 +16,18 @@ const snapshot = (navStatus: PreviewSessionSnapshot["navStatus"]): PreviewSessio }); describe("preview automation open readiness", () => { + it("opens the inline preview by default", () => { + expect(shouldOpenPreviewMiniPlayer({} as PreviewAutomationOpenInput)).toBe(true); + }); + + it("supports explicit opt-out and the legacy show alias", () => { + expect(shouldOpenPreviewMiniPlayer({ open: false } as PreviewAutomationOpenInput)).toBe(false); + expect(shouldOpenPreviewMiniPlayer({ show: false } as PreviewAutomationOpenInput)).toBe(false); + expect( + shouldOpenPreviewMiniPlayer({ open: true, show: false } as PreviewAutomationOpenInput), + ).toBe(true); + }); + it("does not wait for a desktop overlay when opening an empty tab", () => { expect( previewAutomationOpenNeedsOverlay( diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts index 416c2f87c64..1f78c22621f 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts @@ -1,5 +1,9 @@ import type { PreviewAutomationOpenInput, PreviewSessionSnapshot } from "@t3tools/contracts"; +export function shouldOpenPreviewMiniPlayer(input: PreviewAutomationOpenInput): boolean { + return input.open ?? input.show ?? true; +} + export function previewAutomationOpenNeedsOverlay( input: PreviewAutomationOpenInput, snapshot: PreviewSessionSnapshot, diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index 2b321c7eeae..09a13cd31da 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -29,6 +29,16 @@ const decodeAutomationHost = Schema.decodeUnknownSync(PreviewAutomationHost); const decodeAutomationError = Schema.decodeUnknownSync(PreviewAutomationError); const decodeAutomationStatus = Schema.decodeUnknownSync(PreviewAutomationStatus); +describe("PreviewAutomationOpenInput", () => { + it("accepts the inline preview visibility flag", () => { + expect(decodeOpenInput({ open: false })).toEqual({ open: false }); + }); + + it("retains the legacy show visibility alias", () => { + expect(decodeOpenInput({ show: false })).toEqual({ show: false }); + }); +}); + describe("PreviewNavStatus", () => { it("decodes Idle", () => { expect(decodeNavStatus({ _tag: "Idle" })).toEqual({ _tag: "Idle" }); diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index f05623cbc99..d89c4d6f66e 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -86,9 +86,16 @@ export const PreviewAutomationOpenInput = Schema.Struct({ description: "Optional initial page URL, for example https://t3.chat or localhost:5173. Omit to open a blank tab.", }), + open: Schema.optional( + Schema.Boolean.annotate({ + description: + "Whether to open the thread-bound inline preview for the human. Defaults to true; set false for background-only automation.", + }), + ), show: Schema.optional( Schema.Boolean.annotate({ - description: "Whether to reveal the preview panel to the human. Defaults to true.", + description: + "Deprecated alias for open. Whether to reveal the thread-bound inline preview to the human.", }), ), reuseExistingTab: Schema.optional( From ac1bf91160535773ba096d61b5d2825ad2ffe07e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 16:59:06 +0200 Subject: [PATCH 24/26] fix(preview): preserve viewport in inline player Co-authored-by: codex --- .../src/mcp/toolkits/preview/handlers.test.ts | 4 +- .../src/mcp/toolkits/preview/handlers.ts | 4 +- apps/web/src/browser/HostedBrowserWebview.tsx | 43 +++++---- .../src/browser/browserSurfaceStore.test.ts | 29 ++++++ apps/web/src/browser/browserSurfaceStore.ts | 13 ++- .../preview/ThreadPreviewMiniPlayer.tsx | 94 ++++++++++--------- 6 files changed, 123 insertions(+), 64 deletions(-) diff --git a/apps/server/src/mcp/toolkits/preview/handlers.test.ts b/apps/server/src/mcp/toolkits/preview/handlers.test.ts index 4cfdb0c39b6..93985fc9d4c 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.test.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.test.ts @@ -7,6 +7,7 @@ describe("normalizePreviewOpenInput", () => { expect(normalizePreviewOpenInput({})).toEqual({ open: true, reuseExistingTab: true, + show: true, }); }); @@ -14,6 +15,7 @@ describe("normalizePreviewOpenInput", () => { expect(normalizePreviewOpenInput({ open: false })).toEqual({ open: false, reuseExistingTab: true, + show: false, }); }); @@ -26,7 +28,7 @@ describe("normalizePreviewOpenInput", () => { expect(normalizePreviewOpenInput({ open: true, show: false })).toEqual({ open: true, reuseExistingTab: true, - show: false, + show: true, }); }); }); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 484784b911a..1c7ff6f9cd9 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -18,9 +18,11 @@ import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from " export function normalizePreviewOpenInput( input: PreviewAutomationOpenInput, ): PreviewAutomationOpenInput { + const open = input.open ?? input.show ?? true; return { ...input, - open: input.open ?? input.show ?? true, + open, + show: open, reuseExistingTab: input.reuseExistingTab ?? true, }; } diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index 831fe90d629..c3a8271bbf5 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -57,6 +57,7 @@ export function HostedBrowserWebview(props: { const current = state.byTabId[tabId]; return { cornerRadius: current?.cornerRadius ?? 0, + fitSourceContent: current?.fitSourceContent ?? false, fittedSourceContent: current?.fittedSourceContent ?? null, rect: resolveBrowserSurfacePanelRect(state.byTabId, tabId), visible: current?.visible ?? false, @@ -177,26 +178,32 @@ export function HostedBrowserWebview(props: { aspectRatio: lockedAspectRatio, }); const fittedSourceViewport = - presentation.fittedSourceContent && lastRect - ? { - _tag: "freeform" as const, - width: Math.max( - 1, - Math.round( - presentation.fittedSourceContent.width / - presentation.fittedSourceContent.scale / - normalizedZoomFactor, + presentation.fitSourceContent && lastRect + ? presentation.fittedSourceContent + ? { + _tag: "freeform" as const, + width: Math.max( + 1, + Math.round( + presentation.fittedSourceContent.width / + presentation.fittedSourceContent.scale / + normalizedZoomFactor, + ), ), - ), - height: Math.max( - 1, - Math.round( - presentation.fittedSourceContent.height / - presentation.fittedSourceContent.scale / - normalizedZoomFactor, + height: Math.max( + 1, + Math.round( + presentation.fittedSourceContent.height / + presentation.fittedSourceContent.scale / + normalizedZoomFactor, + ), ), - ), - } + } + : { + _tag: "freeform" as const, + width: viewport._tag === "fill" ? 1280 : viewport.width, + height: viewport._tag === "fill" ? 800 : viewport.height, + } : null; const layout = fittedSourceViewport && lastRect diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index dcc3f8ebd69..8a7aa277943 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -40,6 +40,33 @@ describe("browserSurfaceStore", () => { fittedLease.release(); }); + it("freezes the first content dimensions when fitting starts before the browser is measured", () => { + const tabId = "pending-fitted-browser-surface"; + const fittedLease = acquireBrowserSurface(tabId, true); + const sourceContent = { + x: 0, + y: 0, + width: 1_280, + height: 720, + scale: 1, + scrollLeft: 0, + scrollTop: 0, + }; + + useBrowserSurfaceStore.getState().presentContent(tabId, sourceContent); + useBrowserSurfaceStore.getState().presentContent(tabId, { + ...sourceContent, + width: 320, + height: 180, + scale: 0.25, + }); + + expect(useBrowserSurfaceStore.getState().byTabId[tabId]?.fittedSourceContent).toEqual( + sourceContent, + ); + fittedLease.release(); + }); + it("tracks content dimensions for a browser that has never been visible", () => { const tabId = "hidden-browser-surface-content-test"; useBrowserSurfaceStore.getState().presentContent(tabId, { @@ -70,6 +97,7 @@ describe("browserSurfaceStore", () => { visible: false, content: null, fittedSourceContent: null, + fitSourceContent: false, cornerRadius: 0, updatedAt: 1, owner: null, @@ -79,6 +107,7 @@ describe("browserSurfaceStore", () => { visible: true, content: null, fittedSourceContent: null, + fitSourceContent: false, cornerRadius: 0, updatedAt: 2, owner: null, diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index ff7425023b6..ddb69a99d5e 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -12,6 +12,7 @@ export interface BrowserSurfacePresentation { readonly visible: boolean; readonly content: BrowserSurfaceContentPresentation | null; readonly fittedSourceContent: BrowserSurfaceContentPresentation | null; + readonly fitSourceContent: boolean; readonly cornerRadius: number; readonly updatedAt: number; readonly owner: symbol | null; @@ -87,6 +88,7 @@ export const useBrowserSurfaceStore = create()((set) = visible: false, content: current?.content ?? null, fittedSourceContent: fitSourceContent ? (current?.content ?? null) : null, + fitSourceContent, cornerRadius: current?.cornerRadius ?? 0, updatedAt: Date.now(), owner, @@ -125,6 +127,7 @@ export const useBrowserSurfaceStore = create()((set) = visible: false, content, fittedSourceContent: null, + fitSourceContent: false, cornerRadius: 0, updatedAt: Date.now(), owner: null, @@ -148,7 +151,15 @@ export const useBrowserSurfaceStore = create()((set) = return { byTabId: { ...state.byTabId, - [tabId]: { ...current, content, updatedAt: Date.now() }, + [tabId]: { + ...current, + content, + fittedSourceContent: + current.fitSourceContent && current.fittedSourceContent === null + ? content + : current.fittedSourceContent, + updatedAt: Date.now(), + }, }, }; }), diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx index b6e5b372d07..3f1b5dbf4b7 100644 --- a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -227,50 +227,58 @@ export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props } } > -
- - - + + + +
From a6916554e8e7505f3670095e85093917aa98b96a Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 24 Jul 2026 17:04:56 +0200 Subject: [PATCH 25/26] fix(preview): clear fitted surface state Co-authored-by: codex --- apps/web/src/browser/HostedBrowserWebview.tsx | 3 +-- .../src/browser/browserSurfaceStore.test.ts | 23 +++++++++++++++++++ apps/web/src/browser/browserSurfaceStore.ts | 9 +++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index c3a8271bbf5..1d8b1e7db07 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -160,8 +160,7 @@ export function HostedBrowserWebview(props: { } : { width: lastRect?.width ?? 1280, height: lastRect?.height ?? 800 }; const containerSize = active && lastRect ? lastRect : hiddenSize; - const deviceToolbarVisible = - active && viewport._tag !== "fill" && presentation.fittedSourceContent === null; + const deviceToolbarVisible = active && viewport._tag !== "fill" && !presentation.fitSourceContent; const { activeDrag, commitViewportChange, diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index 8a7aa277943..0379f1382e7 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -149,4 +149,27 @@ describe("browserSurfaceStore", () => { owner: null, }); }); + + it("clears fitted presentation state when its lease is released", () => { + const tabId = "released-fitted-browser-surface"; + const fittedLease = acquireBrowserSurface(tabId, true); + useBrowserSurfaceStore.getState().presentContent(tabId, { + x: 0, + y: 0, + width: 1_280, + height: 800, + scale: 1, + scrollLeft: 0, + scrollTop: 0, + }); + + fittedLease.release(); + + expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({ + fittedSourceContent: null, + fitSourceContent: false, + owner: null, + visible: false, + }); + }); }); diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index ddb69a99d5e..c9b7cae544c 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -170,7 +170,14 @@ export const useBrowserSurfaceStore = create()((set) = return { byTabId: { ...state.byTabId, - [tabId]: { ...current, visible: false, updatedAt: Date.now(), owner: null }, + [tabId]: { + ...current, + visible: false, + fittedSourceContent: null, + fitSourceContent: false, + updatedAt: Date.now(), + owner: null, + }, }, }; }), From 00462f44046ecb04c9e833ae59ffacac24e8b310 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 27 Jul 2026 13:06:03 +0200 Subject: [PATCH 26/26] fix(preview): harden capture and PiP races Co-authored-by: codex --- apps/desktop/src/preview/Manager.test.ts | 57 ++++++++++++++++++++++++ apps/desktop/src/preview/Manager.ts | 26 ++++++----- 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 94fdd45550e..cae6c8940d1 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -918,6 +918,62 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("keeps an in-flight frame when a capture consumer is added", () => + withManager((manager) => + Effect.gen(function* () { + const image: TestCapturedPreviewImage = { + toJPEG: vi.fn(() => Buffer.from("shared-in-flight-frame")), + getSize: vi.fn(() => ({ width: 1280, height: 720 })), + }; + let markCaptureStarted!: () => void; + const captureStarted = new Promise((resolve) => { + markCaptureStarted = resolve; + }); + let resolveCapture: ((captured: TestCapturedPreviewImage) => void) | undefined; + const capturePage = vi.fn(() => { + markCaptureStarted(); + return new Promise((resolve) => { + resolveCapture = resolve; + }); + }); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const recordingFrames: DesktopPreviewRecordingFrame[] = []; + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + recordingFrames.push(frame); + }), + ); + + yield* manager.createTab("tab_capture_consumer_added"); + yield* manager.registerWebview("tab_capture_consumer_added", 42); + const recordingFiber = yield* manager + .startRecording("tab_capture_consumer_added") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => captureStarted); + + yield* manager.openPictureInPicture("tab_capture_consumer_added"); + resolveCapture?.(image); + yield* Fiber.join(recordingFiber); + + expect(recordingFrames).toHaveLength(1); + expect(send).toHaveBeenCalledWith( + "desktop:preview-pip-frame", + expect.objectContaining({ + tabId: "tab_capture_consumer_added", + data: Buffer.from("shared-in-flight-frame").toString("base64"), + }), + ); + + yield* manager.stopRecording("tab_capture_consumer_added"); + yield* manager.closePictureInPicture("tab_capture_consumer_added"); + }), + ), + ); + effectIt.effect("shares background frame capture between recording and picture-in-picture", () => withManager((manager) => Effect.gen(function* () { @@ -1160,6 +1216,7 @@ describe("PreviewManager", () => { expect(Exit.hasInterrupts(openExit)).toBe(true); expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(states.some((state) => state.pictureInPicture)).toBe(false); expect(states.at(-1)?.pictureInPicture).toBe(false); }), ), diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index f4b10157079..d8fa674916c 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -2006,18 +2006,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, () => wc.capturePage(), ); - const captureStillCurrent = yield* Effect.all( + const currentCaptureSession = yield* Effect.all( [SynchronizedRef.get(frameCaptureSessionsRef), SynchronizedRef.get(tabsRef)], { concurrency: 2 }, ).pipe( - Effect.map( - ([captureSessions, tabs]) => - captureSessions.get(tabId) === captureSession && + Effect.map(([captureSessions, tabs]) => { + const current = captureSessions.get(tabId); + return current?.scope === captureSession.scope && tabs.get(tabId)?.webContentsId === wc.id && - !wc.isDestroyed(), - ), + !wc.isDestroyed() + ? current + : undefined; + }), ); - if (!captureStillCurrent) return; + if (!currentCaptureSession) return; const size = yield* attempt( { operation: "frameCapture.measureFrame", @@ -2051,7 +2053,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function receivedAt, }; const deliveries: Array> = []; - if (captureSession.consumers.has("recording")) { + if (currentCaptureSession.consumers.has("recording")) { const listeners = yield* Ref.get(recordingFrameListenersRef); deliveries.push( Effect.forEach( @@ -2061,7 +2063,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ); } - if (captureSession.consumers.has("picture-in-picture")) { + if (currentCaptureSession.consumers.has("picture-in-picture")) { const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( tabId, )?.window; @@ -2229,6 +2231,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); if (!pictureInPictureSession) { yield* stopFrameCapture(tabId, "picture-in-picture"); + const tabs = yield* SynchronizedRef.get(tabsRef); + if (tabs.has(tabId)) { + yield* update(tabId, { pictureInPicture: false }); + } return; } yield* releasePictureInPicture(tabId, pictureInPictureSession, true); @@ -2417,7 +2423,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }, () => pictureInPictureSession.window.showInactive(), ); - yield* update(tabId, { pictureInPicture: true }); }); const initializationExit = yield* Effect.gen(function* () { const initializationFiber = yield* Effect.forkIn( @@ -2442,6 +2447,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } return false; } + yield* update(tabId, { pictureInPicture: true }); yield* Deferred.done(pictureInPictureSession.ready, initializationExit); return true; }),