diff --git a/apps/desktop/src/electron/ElectronWindow.test.ts b/apps/desktop/src/electron/ElectronWindow.test.ts index b59f8572739..67819def623 100644 --- a/apps/desktop/src/electron/ElectronWindow.test.ts +++ b/apps/desktop/src/electron/ElectronWindow.test.ts @@ -94,6 +94,7 @@ describe("ElectronWindow", () => { webPreferences: { preload: "/tmp/preload.js", partition: "persist:t3code-preview-test", + backgroundThrottling: null, sandbox: true, contextIsolation: true, nodeIntegration: false, diff --git a/apps/desktop/src/electron/ElectronWindow.ts b/apps/desktop/src/electron/ElectronWindow.ts index dacb2eebb47..4671328587a 100644 --- a/apps/desktop/src/electron/ElectronWindow.ts +++ b/apps/desktop/src/electron/ElectronWindow.ts @@ -22,6 +22,7 @@ const ElectronWindowCreateOptions = Schema.Struct({ webPreferences: Schema.Struct({ preload: Schema.NullOr(Schema.String), partition: Schema.NullOr(Schema.String), + backgroundThrottling: Schema.NullOr(Schema.Boolean), sandbox: Schema.NullOr(Schema.Boolean), contextIsolation: Schema.NullOr(Schema.Boolean), nodeIntegration: Schema.NullOr(Schema.Boolean), @@ -179,6 +180,7 @@ export const make = Effect.gen(function* () { webPreferences: { preload: webPreferences?.preload ?? null, partition: webPreferences?.partition ?? null, + backgroundThrottling: webPreferences?.backgroundThrottling ?? null, sandbox: webPreferences?.sandbox ?? null, contextIsolation: webPreferences?.contextIsolation ?? null, nodeIntegration: webPreferences?.nodeIntegration ?? null, diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 95c725130e5..5988b1e42f9 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -61,6 +61,9 @@ export const PREVIEW_CANCEL_PICK_ELEMENT_CHANNEL = "desktop:preview-cancel-pick- export const PREVIEW_CAPTURE_SCREENSHOT_CHANNEL = "desktop:preview-capture-screenshot"; export const PREVIEW_REVEAL_ARTIFACT_CHANNEL = "desktop:preview-reveal-artifact"; export const PREVIEW_COPY_ARTIFACT_CHANNEL = "desktop:preview-copy-artifact"; +export const PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL = "desktop:preview-pip-open"; +export const PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL = "desktop:preview-pip-close"; +export const PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL = "desktop:preview-pip-frame"; export const PREVIEW_AUTOMATION_STATUS_CHANNEL = "desktop:preview-automation-status"; export const PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL = "desktop:preview-automation-snapshot"; export const PREVIEW_AUTOMATION_CLICK_CHANNEL = "desktop:preview-automation-click"; diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 28405288f6c..4d50ad8d665 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -168,6 +168,16 @@ export const stopRecording = tabMethod( "desktop.ipc.preview.stopRecording", (manager, tabId) => manager.stopRecording(tabId), ); +export const openPictureInPicture = tabMethod( + IpcChannels.PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL, + "desktop.ipc.preview.openPictureInPicture", + (manager, tabId) => manager.openPictureInPicture(tabId), +); +export const closePictureInPicture = tabMethod( + IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, + "desktop.ipc.preview.closePictureInPicture", + (manager, tabId) => manager.closePictureInPicture(tabId), +); export const clearCookies = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_CLEAR_COOKIES_CHANNEL, @@ -367,6 +377,8 @@ export const methods = [ captureScreenshot, revealArtifact, copyArtifactToClipboard, + openPictureInPicture, + closePictureInPicture, automationStatus, automationSnapshot, automationClick, diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 228114fd1d1..9f01baeed90 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -179,6 +179,12 @@ contextBridge.exposeInMainWorld("desktopBridge", { ipcRenderer.invoke(IpcChannels.PREVIEW_REVEAL_ARTIFACT_CHANNEL, { path }), copyArtifactToClipboard: (path) => ipcRenderer.invoke(IpcChannels.PREVIEW_COPY_ARTIFACT_CHANNEL, { path }), + pictureInPicture: { + open: (tabId) => + ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_OPEN_CHANNEL, { tabId }), + close: (tabId) => + ipcRenderer.invoke(IpcChannels.PREVIEW_PICTURE_IN_PICTURE_CLOSE_CHANNEL, { tabId }), + }, recording: { startScreencast: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_RECORDING_START_CHANNEL, { tabId }), diff --git a/apps/desktop/src/preview-pip-preload.ts b/apps/desktop/src/preview-pip-preload.ts new file mode 100644 index 00000000000..384c4129774 --- /dev/null +++ b/apps/desktop/src/preview-pip-preload.ts @@ -0,0 +1,17 @@ +// @effect-diagnostics globalDate:off - This isolated Electron preload does not run inside an Effect runtime. +import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; +import { contextBridge, ipcRenderer } from "electron"; + +import { PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "./ipc/channels.ts"; + +contextBridge.exposeInMainWorld("previewPictureInPicture", { + onFrame: (listener: (frame: DesktopPreviewRecordingFrame) => void) => { + const wrappedListener = (_event: Electron.IpcRendererEvent, frame: unknown) => { + if (typeof frame !== "object" || frame === null) return; + listener(frame as DesktopPreviewRecordingFrame); + }; + ipcRenderer.on(PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, wrappedListener); + return () => + ipcRenderer.removeListener(PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, wrappedListener); + }, +}); diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index f1215ee7b60..cae6c8940d1 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -1,6 +1,8 @@ import { it as effectIt } from "@effect/vitest"; +import type { DesktopPreviewRecordingFrame } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import * as Cause from "effect/Cause"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -19,7 +21,15 @@ import * as ElectronWindow from "../electron/ElectronWindow.ts"; import * as BrowserSession from "./BrowserSession.ts"; import * as PreviewManager from "./Manager.ts"; +describe("fitPictureInPictureContentSize", () => { + it("fits landscape and portrait viewports without letterboxing the window", () => { + expect(PreviewManager.fitPictureInPictureContentSize([480, 320], 16 / 9)).toEqual([480, 270]); + expect(PreviewManager.fitPictureInPictureContentSize([480, 320], 9 / 16)).toEqual([240, 427]); + }); +}); + const { + browserWindowConstructor, createFromPath, fromId, getFocusedWebContents, @@ -29,8 +39,9 @@ const { writeFile, writeImage, } = vi.hoisted(() => ({ + browserWindowConstructor: vi.fn(), createFromPath: vi.fn((): { readonly isEmpty: () => boolean } => ({ isEmpty: () => false })), - fromId: vi.fn(() => null), + fromId: vi.fn((_id?: number) => null), getFocusedWebContents: vi.fn(() => null), mkdir: vi.fn((_path: string) => undefined), showItemInFolder: vi.fn(), @@ -40,6 +51,7 @@ const { })); vi.mock("electron", () => ({ + BrowserWindow: browserWindowConstructor, clipboard: { writeImage, }, @@ -73,6 +85,10 @@ const environmentLayer = Layer.succeed( DesktopEnvironment.DesktopEnvironment, DesktopEnvironment.DesktopEnvironment.of({ browserArtifactsDir: "/tmp/t3/dev/browser-artifacts", + dirname: "/tmp/t3/desktop", + path: { + join: (...parts: ReadonlyArray) => parts.join("/"), + }, } as DesktopEnvironment.DesktopEnvironment["Service"]), ); @@ -92,7 +108,7 @@ const layer = PreviewManager.layer.pipe( Layer.provideMerge(environmentLayer), Layer.provideMerge(fileSystemLayer), Layer.provideMerge(Path.layer), - Layer.provideMerge(Layer.succeed(HostProcessPlatform, "linux")), + Layer.provideMerge(Layer.succeed(HostProcessPlatform, "darwin")), ); const encodePreviewManagerError = Schema.encodeSync(PreviewManager.PreviewManagerError); @@ -106,8 +122,73 @@ const withManager = ( return yield* use(manager); }).pipe(Effect.provide(layer), Effect.scoped); +interface TestCapturedPreviewImage { + readonly toJPEG: () => Buffer; + readonly getSize: () => { readonly width: number; readonly height: number }; +} + +const makeTestPreviewWebContents = ( + capturePage: () => Promise, + id = 42, +) => + ({ + id, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand: vi.fn(async () => undefined), + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + }) as never; + +const makeTestPictureInPictureWindow = (loadURL: () => Promise = async () => undefined) => { + const listeners = new Map void>(); + const send = vi.fn(); + let destroyed = false; + const pictureInPictureWindow = { + isDestroyed: vi.fn(() => destroyed), + once: vi.fn((event: string, listener: () => void) => { + listeners.set(event, listener); + }), + setAlwaysOnTop: vi.fn(), + setVisibleOnAllWorkspaces: vi.fn(), + setAspectRatio: vi.fn(), + getContentSize: vi.fn(() => [480, 320]), + setContentSize: vi.fn(), + loadURL: vi.fn(loadURL), + showInactive: vi.fn(() => { + if (destroyed) throw new Error("Picture-in-picture window is closed."); + }), + close: vi.fn(() => { + if (destroyed) return; + destroyed = true; + listeners.get("closed")?.(); + }), + webContents: { + send, + }, + }; + return { pictureInPictureWindow, send }; +}; + describe("PreviewManager", () => { beforeEach(() => { + browserWindowConstructor.mockReset(); fromId.mockClear(); getFocusedWebContents.mockReset(); getFocusedWebContents.mockReturnValue(null); @@ -438,6 +519,80 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("blocks late webview and capture starts during tab close", () => + withManager((manager) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("close-race-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const firstWebContents = makeTestPreviewWebContents(capturePage, 42); + const replacementWebContents = makeTestPreviewWebContents(capturePage, 43); + const replacementListenerSpies = replacementWebContents as unknown as { + readonly on: ReturnType; + readonly off: ReturnType; + readonly ipc: { readonly off: ReturnType }; + }; + fromId.mockImplementation((id) => { + if (id === 42) return firstWebContents; + if (id === 43) return replacementWebContents; + return null; + }); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_close_register_race"); + yield* manager.registerWebview("tab_close_register_race", 42); + yield* manager.openPictureInPicture("tab_close_register_race"); + + const closeCleanupPaused = yield* Deferred.make(); + const continueCloseCleanup = yield* Deferred.make(); + yield* manager.subscribeStateChanges((_tabId, state) => + !state.pictureInPicture && state.webContentsId === 42 + ? Deferred.succeed(closeCleanupPaused, undefined).pipe( + Effect.andThen(Deferred.await(continueCloseCleanup)), + ) + : Effect.void, + ); + + const closeFiber = yield* manager + .closeTab("tab_close_register_race") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(closeCleanupPaused); + const recreateFiber = yield* manager + .createTab("tab_close_register_race") + .pipe(Effect.forkChild({ startImmediately: true })); + const registrationFiber = yield* manager + .registerWebview("tab_close_register_race", 43) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(replacementListenerSpies.on).not.toHaveBeenCalled(); + yield* manager.closeTab("tab_close_register_race"); + const recordingExit = yield* Effect.exit(manager.startRecording("tab_close_register_race")); + yield* Deferred.succeed(continueCloseCleanup, undefined); + yield* Fiber.join(closeFiber); + const recreated = yield* Fiber.join(recreateFiber); + const registrationExit = yield* Fiber.await(registrationFiber); + + for (const exit of [registrationExit, recordingExit]) { + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isSuccess(exit)) continue; + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewTabNotFoundError", + tabId: "tab_close_register_race", + }); + } + expect(replacementListenerSpies.on).not.toHaveBeenCalled(); + expect(replacementListenerSpies.off).not.toHaveBeenCalled(); + expect(replacementListenerSpies.ipc.off).not.toHaveBeenCalled(); + expect(capturePage).toHaveBeenCalledOnce(); + expect(recreated.webContentsId).toBeNull(); + }), + ), + ); + effectIt.effect("keeps a main-frame load failure visible until a retry starts", () => withManager((manager) => Effect.gen(function* () { @@ -603,6 +758,614 @@ describe("PreviewManager", () => { ), ); + effectIt.effect("captures hidden preview recordings independently for concurrent tabs", () => + withManager((manager) => + Effect.gen(function* () { + const firstJpeg = Buffer.from("first-recording-frame"); + const secondJpeg = Buffer.from("second-recording-frame"); + const firstCapturePage = vi.fn(async () => ({ + toJPEG: () => firstJpeg, + getSize: () => ({ width: 800, height: 600 }), + })); + const secondCapturePage = vi.fn(async () => ({ + toJPEG: () => secondJpeg, + getSize: () => ({ width: 390, height: 844 }), + })); + const firstSendCommand = vi.fn(async () => undefined); + const secondSendCommand = vi.fn(async () => undefined); + const makeWebContents = ( + id: number, + capturePage: typeof firstCapturePage, + sendCommand: typeof firstSendCommand, + ) => + ({ + id, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => `https://example.com/${id}`, + getTitle: () => `Example ${id}`, + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + }) as never; + const webContentsById = new Map([ + [41, makeWebContents(41, firstCapturePage, firstSendCommand)], + [42, makeWebContents(42, secondCapturePage, secondSendCommand)], + ]); + fromId.mockImplementation((id) => + id === undefined ? null : (webContentsById.get(id) ?? null), + ); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_1"); + yield* manager.createTab("tab_2"); + yield* manager.registerWebview("tab_1", 41); + yield* manager.registerWebview("tab_2", 42); + yield* Effect.all([manager.startRecording("tab_1"), manager.startRecording("tab_2")], { + concurrency: 2, + discard: true, + }); + + expect(firstCapturePage).toHaveBeenCalledOnce(); + expect(secondCapturePage).toHaveBeenCalledOnce(); + expect(frames).toHaveLength(2); + expect(frames).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tabId: "tab_1", + data: firstJpeg.toString("base64"), + width: 800, + height: 600, + }), + expect.objectContaining({ + tabId: "tab_2", + data: secondJpeg.toString("base64"), + width: 390, + height: 844, + }), + ]), + ); + expect(firstSendCommand).not.toHaveBeenCalledWith( + "Page.startScreencast", + expect.anything(), + ); + expect(secondSendCommand).not.toHaveBeenCalledWith( + "Page.startScreencast", + expect.anything(), + ); + + yield* Effect.all([manager.stopRecording("tab_1"), manager.stopRecording("tab_2")], { + concurrency: 2, + discard: true, + }); + }), + ), + ); + + effectIt.effect("drops a captured frame when the tab webview changes during capture", () => + withManager((manager) => + Effect.gen(function* () { + const staleImage: TestCapturedPreviewImage = { + toJPEG: vi.fn(() => Buffer.from("stale-recording-frame")), + getSize: vi.fn(() => ({ width: 1280, height: 720 })), + }; + let markCaptureStarted!: () => void; + const captureStarted = new Promise((resolve) => { + markCaptureStarted = resolve; + }); + let resolveCapture: ((image: TestCapturedPreviewImage) => void) | undefined; + const staleCapturePage = vi.fn(() => { + markCaptureStarted(); + return new Promise((resolve) => { + resolveCapture = resolve; + }); + }); + const replacementCapturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("replacement-recording-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const initialWebContents = makeTestPreviewWebContents(staleCapturePage, 42); + const replacementWebContents = makeTestPreviewWebContents(replacementCapturePage, 43); + fromId.mockImplementation((webContentsId?: number) => { + if (webContentsId === 42) return initialWebContents; + if (webContentsId === 43) return replacementWebContents; + return null; + }); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_capture_replaced"); + yield* manager.registerWebview("tab_capture_replaced", 42); + const recordingFiber = yield* manager + .startRecording("tab_capture_replaced") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => captureStarted); + + yield* manager.registerWebview("tab_capture_replaced", 43); + resolveCapture?.(staleImage); + yield* Fiber.join(recordingFiber); + + expect(staleImage.getSize).not.toHaveBeenCalled(); + expect(staleImage.toJPEG).not.toHaveBeenCalled(); + expect(frames).toHaveLength(0); + expect(replacementCapturePage).not.toHaveBeenCalled(); + + yield* manager.stopRecording("tab_capture_replaced"); + }), + ), + ); + + effectIt.effect("keeps an in-flight frame when a capture consumer is added", () => + withManager((manager) => + Effect.gen(function* () { + const image: TestCapturedPreviewImage = { + toJPEG: vi.fn(() => Buffer.from("shared-in-flight-frame")), + getSize: vi.fn(() => ({ width: 1280, height: 720 })), + }; + let markCaptureStarted!: () => void; + const captureStarted = new Promise((resolve) => { + markCaptureStarted = resolve; + }); + let resolveCapture: ((captured: TestCapturedPreviewImage) => void) | undefined; + const capturePage = vi.fn(() => { + markCaptureStarted(); + return new Promise((resolve) => { + resolveCapture = resolve; + }); + }); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const recordingFrames: DesktopPreviewRecordingFrame[] = []; + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + recordingFrames.push(frame); + }), + ); + + yield* manager.createTab("tab_capture_consumer_added"); + yield* manager.registerWebview("tab_capture_consumer_added", 42); + const recordingFiber = yield* manager + .startRecording("tab_capture_consumer_added") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.promise(() => captureStarted); + + yield* manager.openPictureInPicture("tab_capture_consumer_added"); + resolveCapture?.(image); + yield* Fiber.join(recordingFiber); + + expect(recordingFrames).toHaveLength(1); + expect(send).toHaveBeenCalledWith( + "desktop:preview-pip-frame", + expect.objectContaining({ + tabId: "tab_capture_consumer_added", + data: Buffer.from("shared-in-flight-frame").toString("base64"), + }), + ); + + yield* manager.stopRecording("tab_capture_consumer_added"); + yield* manager.closePictureInPicture("tab_capture_consumer_added"); + }), + ), + ); + + effectIt.effect("shares background frame capture between recording and picture-in-picture", () => + withManager((manager) => + Effect.gen(function* () { + const jpeg = Buffer.from("shared-preview-frame"); + const capturePage = vi.fn(async () => ({ + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue({ + id: 42, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + debugger: { + isAttached: () => false, + attach: vi.fn(), + sendCommand: vi.fn(async () => undefined), + on: vi.fn(), + off: vi.fn(), + }, + capturePage, + } as never); + + const pictureInPictureListeners = new Map void>(); + const pictureInPictureSend = vi.fn(); + const pictureInPictureWindow = { + isDestroyed: vi.fn(() => false), + once: vi.fn((event: string, listener: () => void) => { + pictureInPictureListeners.set(event, listener); + }), + setAlwaysOnTop: vi.fn(), + setVisibleOnAllWorkspaces: vi.fn(), + setAspectRatio: vi.fn(), + getContentSize: vi.fn(() => [480, 320] as [number, number]), + setContentSize: vi.fn(), + loadURL: vi.fn(async () => undefined), + showInactive: vi.fn(), + close: vi.fn(() => { + pictureInPictureListeners.get("closed")?.(); + }), + webContents: { + send: pictureInPictureSend, + }, + }; + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const states: PreviewManager.PreviewTabState[] = []; + const recordingFrames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + recordingFrames.push(frame); + }), + ); + yield* manager.createTab("tab_pip"); + yield* manager.registerWebview("tab_pip", 42); + yield* manager.openPictureInPicture("tab_pip"); + + expect(browserWindowConstructor).toHaveBeenCalledWith( + expect.objectContaining({ + alwaysOnTop: true, + show: false, + skipTaskbar: true, + webPreferences: expect.objectContaining({ + preload: "/tmp/t3/desktop/preview-pip-preload.cjs", + backgroundThrottling: false, + }), + }), + ); + expect(pictureInPictureWindow.showInactive).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.setVisibleOnAllWorkspaces).toHaveBeenCalledWith(true, { + visibleOnFullScreen: true, + skipTransformProcessType: true, + }); + expect(pictureInPictureWindow.setAspectRatio.mock.calls).toEqual([[0], [1280 / 720]]); + expect(pictureInPictureWindow.setContentSize).toHaveBeenCalledWith(480, 270, false); + expect(pictureInPictureWindow.setAspectRatio.mock.invocationCallOrder[0]).toBeLessThan( + pictureInPictureWindow.setContentSize.mock.invocationCallOrder[0] ?? 0, + ); + expect(pictureInPictureWindow.setContentSize.mock.invocationCallOrder[0]).toBeLessThan( + pictureInPictureWindow.setAspectRatio.mock.invocationCallOrder[1] ?? 0, + ); + expect(pictureInPictureSend).toHaveBeenCalledWith( + "desktop:preview-pip-frame", + expect.objectContaining({ + tabId: "tab_pip", + data: jpeg.toString("base64"), + width: 1280, + height: 720, + }), + ); + expect(states.at(-1)?.pictureInPicture).toBe(true); + expect(capturePage).toHaveBeenCalledOnce(); + + yield* manager.startRecording("tab_pip"); + expect(capturePage).toHaveBeenCalledOnce(); + expect(recordingFrames).toHaveLength(0); + + yield* TestClock.adjust(100); + expect(capturePage).toHaveBeenCalledTimes(2); + expect(recordingFrames).toHaveLength(1); + + yield* manager.stopRecording("tab_pip"); + const framesBeforePictureInPictureOnlyTick = pictureInPictureSend.mock.calls.length; + yield* TestClock.adjust(100); + expect(capturePage).toHaveBeenCalledTimes(3); + expect(pictureInPictureSend.mock.calls.length).toBeGreaterThan( + framesBeforePictureInPictureOnlyTick, + ); + expect(recordingFrames).toHaveLength(1); + + yield* manager.closePictureInPicture("tab_pip"); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + const capturesAfterClose = capturePage.mock.calls.length; + yield* TestClock.adjust(200); + expect(capturePage).toHaveBeenCalledTimes(capturesAfterClose); + }), + ), + ); + + effectIt.effect("retries a cold hidden-tab capture without dropping recording", () => + withManager((manager) => + Effect.gen(function* () { + const jpeg = Buffer.from("recovered-preview-frame"); + const capturePage = vi.fn(async () => ({ + toJPEG: () => jpeg, + getSize: () => ({ width: 1280, height: 720 }), + })); + capturePage.mockRejectedValueOnce(new Error("UnknownVizError")); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const frames: DesktopPreviewRecordingFrame[] = []; + + yield* manager.subscribeRecordingFrames((frame) => + Effect.sync(() => { + frames.push(frame); + }), + ); + yield* manager.createTab("tab_cold_capture"); + yield* manager.registerWebview("tab_cold_capture", 42); + + yield* manager.startRecording("tab_cold_capture"); + + expect(capturePage).toHaveBeenCalledOnce(); + expect(frames).toHaveLength(0); + + yield* TestClock.adjust(100); + + expect(capturePage).toHaveBeenCalledTimes(2); + expect(frames).toEqual([ + expect.objectContaining({ + tabId: "tab_cold_capture", + data: jpeg.toString("base64"), + width: 1280, + height: 720, + }), + ]); + + yield* manager.stopRecording("tab_cold_capture"); + }), + ), + ); + + effectIt.effect("drops empty frames before picture-in-picture delivery", () => + withManager((manager) => + Effect.gen(function* () { + const validImage: TestCapturedPreviewImage = { + toJPEG: () => Buffer.from("valid-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + }; + const capturePage = vi.fn(async () => validImage); + capturePage.mockResolvedValueOnce({ + toJPEG: () => Buffer.from("empty-preview-frame"), + getSize: () => ({ width: 0, height: 0 }), + }); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow(); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_empty_frame"); + yield* manager.registerWebview("tab_empty_frame", 42); + yield* manager.openPictureInPicture("tab_empty_frame"); + + expect(capturePage).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.setAspectRatio).not.toHaveBeenCalled(); + expect(send).not.toHaveBeenCalled(); + + yield* TestClock.adjust(100); + + expect(pictureInPictureWindow.setAspectRatio.mock.calls).toEqual([[0], [1280 / 720]]); + expect(send).toHaveBeenCalledOnce(); + yield* manager.closePictureInPicture("tab_empty_frame"); + }), + ), + ); + + effectIt.effect("does not publish picture-in-picture readiness after window teardown", () => + withManager((manager) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("closing-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow } = makeTestPictureInPictureWindow(); + pictureInPictureWindow.showInactive.mockImplementationOnce(() => { + pictureInPictureWindow.close(); + }); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + + yield* manager.createTab("tab_pip_teardown"); + yield* manager.registerWebview("tab_pip_teardown", 42); + const openExit = yield* Effect.exit(manager.openPictureInPicture("tab_pip_teardown")); + + expect(Exit.hasInterrupts(openExit)).toBe(true); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(states.some((state) => state.pictureInPicture)).toBe(false); + expect(states.at(-1)?.pictureInPicture).toBe(false); + }), + ), + ); + + effectIt.effect("closes an initializing picture-in-picture without blocking later opens", () => + withManager((manager) => + Effect.gen(function* () { + const capturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("serialized-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + fromId.mockReturnValue(makeTestPreviewWebContents(capturePage)); + const { pictureInPictureWindow: initializingWindow } = makeTestPictureInPictureWindow( + () => + new Promise(() => { + // Simulate a renderer load that never settles. + }), + ); + const { pictureInPictureWindow: reopenedWindow } = makeTestPictureInPictureWindow(); + browserWindowConstructor + .mockImplementationOnce(function () { + return initializingWindow; + }) + .mockImplementationOnce(function () { + return reopenedWindow; + }); + const states: PreviewManager.PreviewTabState[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + states.push(state); + }), + ); + yield* manager.createTab("tab_concurrent_pip"); + yield* manager.registerWebview("tab_concurrent_pip", 42); + + const firstOpen = yield* manager + .openPictureInPicture("tab_concurrent_pip") + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const secondOpen = yield* manager + .openPictureInPicture("tab_concurrent_pip") + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + const close = yield* manager + .closePictureInPicture("tab_concurrent_pip") + .pipe(Effect.forkChild); + yield* Effect.yieldNow; + + expect(browserWindowConstructor).toHaveBeenCalledOnce(); + expect(initializingWindow.loadURL).toHaveBeenCalledOnce(); + expect(initializingWindow.close).toHaveBeenCalledOnce(); + const [firstOpenExit, secondOpenExit] = yield* Effect.all([ + Fiber.await(firstOpen), + Fiber.await(secondOpen), + ]); + yield* Fiber.join(close); + + expect(Exit.hasInterrupts(firstOpenExit)).toBe(true); + expect(Exit.hasInterrupts(secondOpenExit)).toBe(true); + expect(initializingWindow.showInactive).not.toHaveBeenCalled(); + expect(capturePage).not.toHaveBeenCalled(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + + yield* manager.openPictureInPicture("tab_concurrent_pip"); + + expect(browserWindowConstructor).toHaveBeenCalledTimes(2); + expect(reopenedWindow.showInactive).toHaveBeenCalledOnce(); + expect(capturePage).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(true); + + yield* manager.closePictureInPicture("tab_concurrent_pip"); + + expect(browserWindowConstructor).toHaveBeenCalledTimes(2); + expect(reopenedWindow.close).toHaveBeenCalledOnce(); + expect(states.at(-1)?.pictureInPicture).toBe(false); + const capturesAfterClose = capturePage.mock.calls.length; + yield* TestClock.adjust(200); + expect(capturePage).toHaveBeenCalledTimes(capturesAfterClose); + }), + ), + ); + + effectIt.effect("rejects picture-in-picture when its webview changes during initialization", () => + withManager((manager) => + Effect.gen(function* () { + const initialCapturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("stale-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const replacementCapturePage = vi.fn(async () => ({ + toJPEG: () => Buffer.from("replacement-preview-frame"), + getSize: () => ({ width: 1280, height: 720 }), + })); + const initialWebContents = makeTestPreviewWebContents(initialCapturePage, 42); + const replacementWebContents = makeTestPreviewWebContents(replacementCapturePage, 43); + fromId.mockImplementation((webContentsId?: number) => { + if (webContentsId === 42) return initialWebContents; + if (webContentsId === 43) return replacementWebContents; + return null; + }); + let resolveLoad: (() => void) | undefined; + const { pictureInPictureWindow } = makeTestPictureInPictureWindow( + () => + new Promise((resolve) => { + resolveLoad = resolve; + }), + ); + browserWindowConstructor.mockImplementation(function () { + return pictureInPictureWindow; + }); + + yield* manager.createTab("tab_replaced_webview"); + yield* manager.registerWebview("tab_replaced_webview", 42); + const open = yield* manager + .openPictureInPicture("tab_replaced_webview") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(pictureInPictureWindow.loadURL).toHaveBeenCalledOnce(); + expect(resolveLoad).toBeDefined(); + const concurrentOpen = yield* manager + .openPictureInPicture("tab_replaced_webview") + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + + yield* manager.registerWebview("tab_replaced_webview", 43); + resolveLoad?.(); + + const openExits = yield* Effect.all([Fiber.await(open), Fiber.await(concurrentOpen)]); + for (const openExit of openExits) { + expect(Exit.isFailure(openExit)).toBe(true); + if (Exit.isSuccess(openExit)) continue; + const error = Option.getOrThrow(Cause.findErrorOption(openExit.cause)); + expect(error).toMatchObject({ + _tag: "PreviewOperationError", + operation: "pictureInPicture.validateWebContents", + tabId: "tab_replaced_webview", + webContentsId: 42, + }); + } + expect(browserWindowConstructor).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.close).toHaveBeenCalledOnce(); + expect(pictureInPictureWindow.showInactive).not.toHaveBeenCalled(); + expect(initialCapturePage).not.toHaveBeenCalled(); + expect(replacementCapturePage).not.toHaveBeenCalled(); + }), + ), + ); + effectIt.effect("keeps element picking active during subframe navigation", () => withManager((manager) => Effect.gen(function* () { diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 6c942d4ccb9..d8fa674916c 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -28,21 +28,16 @@ import type { } from "@t3tools/contracts"; import { HostProcessPlatform } from "@t3tools/shared/hostProcess"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; -import { - type BrowserWindow, - type Session, - clipboard, - nativeImage, - shell, - webContents, -} from "electron"; +import { BrowserWindow, type Session, clipboard, nativeImage, shell, webContents } from "electron"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; @@ -53,6 +48,7 @@ import * as Scope from "effect/Scope"; import * as SynchronizedRef from "effect/SynchronizedRef"; import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; +import { PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL } from "../ipc/channels.ts"; import * as BrowserSession from "./BrowserSession.ts"; import { ANNOTATION_CAPTURED_CHANNEL, @@ -85,6 +81,7 @@ export interface PreviewTabState { canGoBack: boolean; canGoForward: boolean; zoomFactor: number; + pictureInPicture: boolean; colorScheme: DesktopPreviewColorScheme; controller: "human" | "agent" | "none"; updatedAt: string; @@ -101,6 +98,12 @@ const MAX_EVALUATION_BYTES = 64_000; const MAX_VISIBLE_TEXT_LENGTH = 20_000; const MAX_INTERACTIVE_ELEMENTS = 200; const MAX_SCREENSHOT_WIDTH = 1280; +const RECORDING_FRAME_INTERVAL_MS = Math.ceil(1_000 / 12); +const RECORDING_JPEG_QUALITY = 80; +const PICTURE_IN_PICTURE_INITIAL_WIDTH = 480; +const PICTURE_IN_PICTURE_INITIAL_HEIGHT = 320; +const PICTURE_IN_PICTURE_MIN_WIDTH = 240; +const PICTURE_IN_PICTURE_MIN_HEIGHT = 160; const DIAGNOSTIC_BUFFER_LIMIT = 200; const MAX_ARTIFACT_SITE_SLUG_LENGTH = 80; const AGENT_CURSOR_MOVE_MS = 160; @@ -126,6 +129,54 @@ const DEFAULT_ANNOTATION_THEME: DesktopPreviewAnnotationTheme = { fontMono: "ui-monospace, monospace", }; +export const buildPreviewPictureInPictureDataUrl = (): string => { + const html = ` + + + + + + + + + Live browser preview + + +`; + return `data:text/html;charset=utf-8,${encodeURIComponent(html)}`; +}; + +export const fitPictureInPictureContentSize = ( + current: ReadonlyArray, + aspectRatio: number, +): readonly [width: number, height: number] => { + const currentWidth = Math.max(1, current[0] ?? PICTURE_IN_PICTURE_INITIAL_WIDTH); + const currentHeight = Math.max(1, current[1] ?? PICTURE_IN_PICTURE_INITIAL_HEIGHT); + let width = + currentWidth / currentHeight > aspectRatio ? currentHeight * aspectRatio : currentWidth; + let height = width / aspectRatio; + const minimumScale = Math.max( + 1, + PICTURE_IN_PICTURE_MIN_WIDTH / width, + PICTURE_IN_PICTURE_MIN_HEIGHT / height, + ); + width *= minimumScale; + height *= minimumScale; + return [Math.round(width), Math.round(height)]; +}; + const artifactSiteSlug = (rawUrl: string): string => { try { const url = new URL(rawUrl); @@ -296,6 +347,20 @@ interface ManagedListeners { readonly scope: Scope.Closeable; } +type FrameCaptureConsumer = "picture-in-picture" | "recording"; + +interface FrameCaptureSession { + readonly scope: Scope.Closeable; + readonly consumers: ReadonlySet; +} + +interface PictureInPictureSession { + readonly window: BrowserWindow; + readonly webContentsId: number; + readonly ready: Deferred.Deferred; + readonly initializationScope: Scope.Closeable; +} + interface PickSession { readonly cancel: Effect.Effect; } @@ -380,6 +445,7 @@ const inputSignalsMatch = (left: PreviewInputSignal, right: PreviewInputSignal): const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function* ( artifactDirectory: string, + pictureInPicturePreloadPath: string, ) { const fileSystem = yield* FileSystem.FileSystem; const hostPlatform = yield* HostProcessPlatform; @@ -415,7 +481,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function >(new Map()); const actionSequenceRef = yield* Ref.make(0); const pointerSequenceRef = yield* Ref.make(0); - const recordingTabIdRef = yield* Ref.make>(Option.none()); + const frameCaptureSessionsRef = yield* SynchronizedRef.make< + ReadonlyMap + >(new Map()); + const pictureInPictureSessionsRef = yield* SynchronizedRef.make< + ReadonlyMap + >(new Map()); + const pictureInPictureAspectRatiosRef = yield* Ref.make>(new Map()); + const pictureInPictureMutationSemaphore = yield* Semaphore.make(1); + const closingTabIdsRef = yield* Ref.make>(new Set()); + const tabLifecycleLocks = new Map< + string, + { readonly semaphore: Semaphore.Semaphore; users: number } + >(); + const tabLifecycleGenerations = new Map(); const attempt = (errorContext: PreviewOperationContext, evaluate: () => A) => Effect.try({ @@ -446,6 +525,58 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function update(copy); return copy; }; + const withTabLifecycleLock = ( + tabId: string, + effect: Effect.Effect, + ): Effect.Effect => + Effect.suspend(() => { + const lifecycle = tabLifecycleLocks.get(tabId) ?? { + semaphore: Semaphore.makeUnsafe(1), + users: 0, + }; + lifecycle.users += 1; + tabLifecycleLocks.set(tabId, lifecycle); + return lifecycle.semaphore.withPermit(effect).pipe( + Effect.ensuring( + Effect.sync(() => { + lifecycle.users -= 1; + if (lifecycle.users === 0 && tabLifecycleLocks.get(tabId) === lifecycle) { + tabLifecycleLocks.delete(tabId); + } + }), + ), + ); + }); + const stopFrameCapture = Effect.fn("PreviewManager.stopFrameCapture")(function* ( + tabId: string, + consumer: FrameCaptureConsumer, + ) { + const captureScope = yield* SynchronizedRef.modify(frameCaptureSessionsRef, (sessions) => { + const current = sessions.get(tabId); + if (!current || !current.consumers.has(consumer)) { + return [undefined, sessions] as const; + } + const consumers = new Set(current.consumers); + consumers.delete(consumer); + if (consumers.size > 0) { + return [ + undefined, + replaceMap(sessions, (copy) => { + copy.set(tabId, { ...current, consumers }); + }), + ] as const; + } + return [ + current.scope, + replaceMap(sessions, (copy) => { + copy.delete(tabId); + }), + ] as const; + }); + if (captureScope) { + yield* Scope.close(captureScope, Exit.void).pipe(Effect.ignore); + } + }); const deliverEvent = ( eventKind: "state-change" | "recording-frame" | "pointer-event", @@ -539,15 +670,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ), ); - const tabIdForWebContents = Effect.fn("PreviewManager.tabIdForWebContents")(function* ( - webContentsId: number, - ) { - const tabs = yield* SynchronizedRef.get(tabsRef); - return ( - Array.from(tabs.entries()).find(([, tab]) => tab.webContentsId === webContentsId)?.[0] ?? null - ); - }); - const pushBounded = (buffer: ReadonlyArray, entry: A): ReadonlyArray => [...buffer, entry].slice(-DIAGNOSTIC_BUFFER_LIMIT); @@ -731,42 +853,6 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const scope = yield* Scope.fork(parentScope, "sequential"); const handleDebuggerMessage = Effect.fn("PreviewManager.handleDebuggerMessage")( function* (method: string, params: Record) { - if (method === "Page.screencastFrame") { - const sessionId = params["sessionId"]; - if (typeof sessionId === "number") { - yield* attemptPromise( - { - operation: "ackScreencastFrame", - webContentsId: wc.id, - }, - () => wc.debugger.sendCommand("Page.screencastFrameAck", { sessionId }), - ).pipe(Effect.ignore); - } - const tabId = yield* tabIdForWebContents(wc.id); - const metadata = - typeof params["metadata"] === "object" && params["metadata"] !== null - ? (params["metadata"] as Record) - : {}; - if (tabId && typeof params["data"] === "string") { - const receivedAt = yield* currentIso; - const listeners = yield* Ref.get(recordingFrameListenersRef); - const frame: DesktopPreviewRecordingFrame = { - tabId, - data: params["data"], - width: - typeof metadata["deviceWidth"] === "number" ? metadata["deviceWidth"] : 0, - height: - typeof metadata["deviceHeight"] === "number" ? metadata["deviceHeight"] : 0, - receivedAt, - }; - yield* Effect.forEach( - listeners, - (listener) => - deliverEvent("recording-frame", frame.tabId, () => listener(frame)), - { discard: true }, - ); - } - } yield* captureDiagnosticMessage(wc.id, method, params); }, ); @@ -1276,71 +1362,132 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function window: BrowserWindow, ) { yield* Ref.set(mainWindowRef, Option.some(window)); + window.once("closed", () => { + runFork(closeAllPictureInPicture()); + }); }); - const createTab = Effect.fn("PreviewManager.createTab")(function* (tabId: string) { + const createTabUnlocked = Effect.fn("PreviewManager.createTabUnlocked")(function* ( + tabId: string, + ) { const updatedAt = yield* currentIso; - const state = yield* SynchronizedRef.modify(tabsRef, (tabs) => { - const existing = tabs.get(tabId); - if (existing) return [existing, tabs] as const; - const initial: PreviewTabState = { - tabId, - webContentsId: null, - navStatus: { kind: "Idle" }, - canGoBack: false, - canGoForward: false, - zoomFactor: DEFAULT_ZOOM_FACTOR, - colorScheme: "system", - controller: "none", - updatedAt, - }; + const result = yield* SynchronizedRef.modify( + tabsRef, + ( + tabs, + ): readonly [ + { readonly state: PreviewTabState; readonly created: boolean }, + ReadonlyMap, + ] => { + const existing = tabs.get(tabId); + if (existing) return [{ state: existing, created: false }, tabs] as const; + const initial: PreviewTabState = { + tabId, + webContentsId: null, + navStatus: { kind: "Idle" }, + canGoBack: false, + canGoForward: false, + zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, + colorScheme: "system", + controller: "none", + updatedAt, + }; + return [ + { state: initial, created: true }, + replaceMap(tabs, (copy) => { + copy.set(tabId, initial); + }), + ] as const; + }, + ); + if (result.created) { + tabLifecycleGenerations.set(tabId, (tabLifecycleGenerations.get(tabId) ?? 0) + 1); + } + yield* emit(tabId, result.state); + return result.state; + }); + + const createTab = Effect.fn("PreviewManager.createTab")(function* (tabId: string) { + return yield* withTabLifecycleLock(tabId, createTabUnlocked(tabId)); + }); + + const closeTabUnlocked = Effect.fn("PreviewManager.closeTabUnlocked")(function* (tabId: string) { + if (!(yield* SynchronizedRef.get(tabsRef)).has(tabId)) return; + yield* Effect.all( + [ + cancelPickElement(tabId), + closePictureInPicture(tabId), + stopFrameCapture(tabId, "recording"), + ], + { + concurrency: 3, + discard: true, + }, + ); + const tab = yield* SynchronizedRef.modify(tabsRef, (tabs) => { + const current = tabs.get(tabId); + if (!current) return [Option.none(), tabs] as const; return [ - initial, + Option.some(current), replaceMap(tabs, (copy) => { - copy.set(tabId, initial); + copy.delete(tabId); }), ] as const; }); - yield* emit(tabId, state); - return state; - }); - - const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { - const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (!tab) return; - yield* cancelPickElement(tabId); - if (tab.webContentsId != null) { + if (Option.isNone(tab)) return; + const closedTab = tab.value; + if (closedTab.webContentsId != null) { yield* Effect.all( - [detachControlSession(tab.webContentsId), detachListeners(tab.webContentsId)], + [detachControlSession(closedTab.webContentsId), detachListeners(closedTab.webContentsId)], { concurrency: 2, discard: true }, ); } const updatedAt = yield* currentIso; const closed: PreviewTabState = { - ...tab, + ...closedTab, webContentsId: null, navStatus: { kind: "Idle" }, canGoBack: false, canGoForward: false, zoomFactor: DEFAULT_ZOOM_FACTOR, + pictureInPicture: false, colorScheme: "system", controller: "none", updatedAt, }; - yield* SynchronizedRef.update(tabsRef, (tabs) => - replaceMap(tabs, (copy) => { - copy.delete(tabId); - }), - ); yield* emit(tabId, closed); }); - const registerWebview = Effect.fn("PreviewManager.registerWebview")(function* ( + const closeTab = Effect.fn("PreviewManager.closeTab")(function* (tabId: string) { + const claimed = yield* Ref.modify(closingTabIdsRef, (closingTabIds) => { + if (closingTabIds.has(tabId)) return [false, closingTabIds] as const; + return [true, new Set([...closingTabIds, tabId])] as const; + }); + if (!claimed) return; + return yield* withTabLifecycleLock(tabId, closeTabUnlocked(tabId)).pipe( + Effect.ensuring( + Ref.update(closingTabIdsRef, (closingTabIds) => { + if (!closingTabIds.has(tabId)) return closingTabIds; + const next = new Set(closingTabIds); + next.delete(tabId); + return next; + }), + ), + ); + }); + + const registerWebviewUnlocked = Effect.fn("PreviewManager.registerWebviewUnlocked")(function* ( tabId: string, webContentsId: number, + expectedGeneration: number | undefined, ) { const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); - if (!tab) { + if ( + !tab || + tabLifecycleGenerations.get(tabId) !== expectedGeneration || + (yield* Ref.get(closingTabIdsRef)).has(tabId) + ) { return yield* new PreviewTabNotFoundError({ tabId }); } const wc = webContents.fromId(webContentsId); @@ -1377,53 +1524,71 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { concurrency: 3, discard: true }, ); } + const currentTab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if ( + !currentTab || + tabLifecycleGenerations.get(tabId) !== expectedGeneration || + (yield* Ref.get(closingTabIdsRef)).has(tabId) + ) { + return yield* new PreviewTabNotFoundError({ tabId }); + } const zoomFactor = replacedWebContentsId !== null ? yield* attempt( { operation: "registerWebview.restoreZoomFactor", tabId, webContentsId }, () => { - wc.setZoomFactor(tab.zoomFactor); - return tab.zoomFactor; + wc.setZoomFactor(currentTab.zoomFactor); + return currentTab.zoomFactor; }, ) : yield* attempt({ operation: "registerWebview.getZoomFactor", tabId, webContentsId }, () => wc.getZoomFactor(), ); yield* attachListeners(tabId, wc); - runFork(restoreControlSession(tabId, wc)); const registeredAt = yield* currentIso; - const registration = yield* SynchronizedRef.modify(tabsRef, (tabs) => { - const current = tabs.get(tabId); - if (!current) { + const registration = yield* SynchronizedRef.modifyEffect(tabsRef, (tabs) => + Effect.gen(function* () { + const current = tabs.get(tabId); + if ( + !current || + tabLifecycleGenerations.get(tabId) !== expectedGeneration || + (yield* Ref.get(closingTabIdsRef)).has(tabId) + ) { + return [ + Option.none<{ readonly state: PreviewTabState; readonly pendingUrl: string | null }>(), + tabs, + ] as const; + } + const pendingUrl = current.navStatus.kind === "Loading" ? current.navStatus.url : null; + const next: PreviewTabState = { + ...current, + webContentsId, + navStatus: pendingUrl === null ? computeNavStatus(wc) : current.navStatus, + canGoBack: wc.navigationHistory.canGoBack(), + canGoForward: wc.navigationHistory.canGoForward(), + zoomFactor, + updatedAt: registeredAt, + }; return [ - Option.none<{ readonly state: PreviewTabState; readonly pendingUrl: string | null }>(), - tabs, + Option.some({ + state: next, + pendingUrl, + }), + replaceMap(tabs, (copy) => { + copy.set(tabId, next); + }), ] as const; - } - const pendingUrl = current.navStatus.kind === "Loading" ? current.navStatus.url : null; - const next: PreviewTabState = { - ...current, - webContentsId, - navStatus: pendingUrl === null ? computeNavStatus(wc) : current.navStatus, - canGoBack: wc.navigationHistory.canGoBack(), - canGoForward: wc.navigationHistory.canGoForward(), - zoomFactor, - updatedAt: registeredAt, - }; - return [ - Option.some({ - state: next, - pendingUrl, - }), - replaceMap(tabs, (copy) => { - copy.set(tabId, next); - }), - ] as const; - }); + }), + ); if (Option.isNone(registration)) { + yield* Effect.all([detachControlSession(webContentsId), detachListeners(webContentsId)], { + concurrency: 2, + discard: true, + }); return yield* new PreviewTabNotFoundError({ tabId }); } const { state: registered, pendingUrl } = registration.value; + runFork(restoreControlSession(tabId, wc)); yield* emit(tabId, registered); yield* attempt({ operation: "registerWebview.sendTheme", tabId, webContentsId }, () => wc.send(ANNOTATION_THEME_CHANNEL, annotationTheme), @@ -1443,6 +1608,17 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } }); + const registerWebview = Effect.fn("PreviewManager.registerWebview")(function* ( + tabId: string, + webContentsId: number, + ) { + const expectedGeneration = tabLifecycleGenerations.get(tabId); + return yield* withTabLifecycleLock( + tabId, + registerWebviewUnlocked(tabId, webContentsId, expectedGeneration), + ); + }); + const navigate = Effect.fn("PreviewManager.navigate")(function* (tabId: string, rawUrl: string) { const url = yield* attempt({ operation: "navigate.normalizeUrl", tabId }, () => normalizePreviewUrl(rawUrl), @@ -1461,6 +1637,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function canGoBack: current?.canGoBack ?? false, canGoForward: current?.canGoForward ?? false, zoomFactor: current?.zoomFactor ?? DEFAULT_ZOOM_FACTOR, + pictureInPicture: current?.pictureInPicture ?? false, colorScheme: current?.colorScheme ?? "system", controller: current?.controller ?? "none", updatedAt, @@ -1716,14 +1893,28 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function // session attaches so a concurrent setColorScheme is not overwritten with // a stale snapshot. const restoreControlSession = (tabId: string, wc: Electron.WebContents) => - ensureControlSession(wc).pipe( - Effect.andThen(SynchronizedRef.get(tabsRef)), - Effect.flatMap((tabs) => { - const colorScheme = tabs.get(tabId)?.colorScheme ?? "system"; - return colorScheme === "system" ? Effect.void : applyColorScheme(tabId, wc, colorScheme); - }), - Effect.ignore, - ); + Effect.gen(function* () { + const beforeAttach = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (beforeAttach?.webContentsId !== wc.id) return; + yield* ensureControlSession(wc); + const afterAttach = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (afterAttach?.webContentsId !== wc.id) { + yield* detachControlSession(wc.id); + return; + } + if (afterAttach.colorScheme !== "system") { + yield* attemptPromise({ operation: "applyColorScheme", tabId, webContentsId: wc.id }, () => + wc.debugger.sendCommand("Emulation.setEmulatedMedia", { + features: [ + { + name: "prefers-color-scheme", + value: afterAttach.colorScheme, + }, + ], + }), + ); + } + }).pipe(Effect.ignore); const setColorScheme = Effect.fn("PreviewManager.setColorScheme")(function* ( tabId: string, @@ -1801,40 +1992,485 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; }); - const startScreencast = Effect.fn("PreviewManager.startScreencast")(function* ( - send: SendCommand, + const capturePreviewFrame = Effect.fn("PreviewManager.capturePreviewFrame")(function* ( + tabId: string, ) { - yield* send("Page.enable"); - yield* send("Page.startScreencast", { - format: "jpeg", - quality: 80, - maxWidth: 1600, - maxHeight: 1200, - everyNthFrame: 1, + const captureSession = (yield* SynchronizedRef.get(frameCaptureSessionsRef)).get(tabId); + if (!captureSession) return; + const wc = yield* requireWebContents(tabId); + const image = yield* attemptPromise( + { + operation: "frameCapture.capturePage", + tabId, + webContentsId: wc.id, + }, + () => wc.capturePage(), + ); + const currentCaptureSession = yield* Effect.all( + [SynchronizedRef.get(frameCaptureSessionsRef), SynchronizedRef.get(tabsRef)], + { concurrency: 2 }, + ).pipe( + Effect.map(([captureSessions, tabs]) => { + const current = captureSessions.get(tabId); + return current?.scope === captureSession.scope && + tabs.get(tabId)?.webContentsId === wc.id && + !wc.isDestroyed() + ? current + : undefined; + }), + ); + if (!currentCaptureSession) return; + const size = yield* attempt( + { + operation: "frameCapture.measureFrame", + tabId, + webContentsId: wc.id, + }, + () => image.getSize(), + ); + if ( + !Number.isFinite(size.width) || + !Number.isFinite(size.height) || + size.width <= 0 || + size.height <= 0 + ) { + return; + } + const encoded = yield* attempt( + { + operation: "frameCapture.encodeFrame", + tabId, + webContentsId: wc.id, + }, + () => image.toJPEG(RECORDING_JPEG_QUALITY).toString("base64"), + ); + const receivedAt = yield* currentIso; + const frame: DesktopPreviewRecordingFrame = { + tabId, + data: encoded, + width: size.width, + height: size.height, + receivedAt, + }; + const deliveries: Array> = []; + if (currentCaptureSession.consumers.has("recording")) { + const listeners = yield* Ref.get(recordingFrameListenersRef); + deliveries.push( + Effect.forEach( + listeners, + (listener) => deliverEvent("recording-frame", frame.tabId, () => listener(frame)), + { discard: true }, + ), + ); + } + if (currentCaptureSession.consumers.has("picture-in-picture")) { + const pictureInPictureWindow = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( + tabId, + )?.window; + if (pictureInPictureWindow && !pictureInPictureWindow.isDestroyed()) { + deliveries.push( + Effect.gen(function* () { + const previousAspectRatio = (yield* Ref.get(pictureInPictureAspectRatiosRef)).get( + tabId, + ); + const aspectRatio = frame.width / frame.height; + if (previousAspectRatio !== aspectRatio) { + yield* attempt( + { + operation: "pictureInPicture.setAspectRatio", + tabId, + webContentsId: wc.id, + }, + () => { + const contentSize = fitPictureInPictureContentSize( + pictureInPictureWindow.getContentSize(), + aspectRatio, + ); + pictureInPictureWindow.setAspectRatio(0); + pictureInPictureWindow.setContentSize(contentSize[0], contentSize[1], false); + pictureInPictureWindow.setAspectRatio(aspectRatio); + }, + ); + yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => + replaceMap(aspectRatios, (copy) => { + copy.set(tabId, aspectRatio); + }), + ); + } + yield* attempt( + { + operation: "pictureInPicture.deliverFrame", + tabId, + webContentsId: wc.id, + }, + () => { + pictureInPictureWindow.webContents.send( + PREVIEW_PICTURE_IN_PICTURE_FRAME_CHANNEL, + frame, + ); + }, + ); + }).pipe( + Effect.catch((error) => + Effect.logWarning("Picture-in-picture frame delivery failed.", { + tabId, + error, + }), + ), + ), + ); + } + } + yield* Effect.all(deliveries, { concurrency: 2, discard: true }); + }); + + const startFrameCapture = Effect.fn("PreviewManager.startFrameCapture")(function* ( + tabId: string, + consumer: FrameCaptureConsumer, + ) { + // Validate the tab synchronously, but treat capturePage failures as + // transient. Chromium can return UnknownVizError while a hidden guest is + // warming its first compositor frame; the scheduled loop should keep the + // consumer alive and recover instead of tearing recording/PiP back down. + yield* requireWebContents(tabId); + const captureNextFrame = Effect.sleep(RECORDING_FRAME_INTERVAL_MS).pipe( + Effect.andThen(capturePreviewFrame(tabId)), + Effect.catch((error) => + Effect.logWarning("Background preview frame capture failed.", { + tabId, + error, + }), + ), + ); + const created = yield* SynchronizedRef.modifyEffect(frameCaptureSessionsRef, (sessions) => { + return Effect.gen(function* () { + const tab = (yield* SynchronizedRef.get(tabsRef)).get(tabId); + if (!tab || (yield* Ref.get(closingTabIdsRef)).has(tabId)) { + return yield* new PreviewTabNotFoundError({ tabId }); + } + const current = sessions.get(tabId); + if (current) { + if (current.consumers.has(consumer)) { + return [false, sessions] as const; + } + return [ + false, + replaceMap(sessions, (copy) => { + copy.set(tabId, { + ...current, + consumers: new Set([...current.consumers, consumer]), + }); + }), + ] as const; + } + const scope = yield* Scope.fork(parentScope, "sequential"); + yield* Effect.forkIn(Effect.forever(captureNextFrame), scope); + return [ + true, + replaceMap(sessions, (copy) => { + copy.set(tabId, { + scope, + consumers: new Set([consumer]), + }); + }), + ] as const; + }); + }); + if (!created) return; + yield* capturePreviewFrame(tabId).pipe( + Effect.catch((error) => + Effect.logWarning("Initial background preview frame was not ready; capture will retry.", { + tabId, + consumer, + error, + }), + ), + ); + }); + + const releasePictureInPicture = Effect.fn("PreviewManager.releasePictureInPicture")(function* ( + tabId: string, + expectedSession: PictureInPictureSession, + closeWindow: boolean, + ) { + const removed = yield* SynchronizedRef.modify(pictureInPictureSessionsRef, (sessions) => { + if (sessions.get(tabId) !== expectedSession) { + return [false, sessions] as const; + } + return [ + true, + replaceMap(sessions, (copy) => { + copy.delete(tabId); + }), + ] as const; }); + if (!removed) return; + yield* Deferred.interrupt(expectedSession.ready); + yield* Scope.close(expectedSession.initializationScope, Exit.void).pipe(Effect.ignore); + yield* Ref.update(pictureInPictureAspectRatiosRef, (aspectRatios) => + replaceMap(aspectRatios, (copy) => { + copy.delete(tabId); + }), + ); + yield* stopFrameCapture(tabId, "picture-in-picture"); + const tabs = yield* SynchronizedRef.get(tabsRef); + if (tabs.has(tabId)) { + yield* update(tabId, { pictureInPicture: false }); + } + if (closeWindow && !expectedSession.window.isDestroyed()) { + yield* attempt({ operation: "pictureInPicture.close", tabId }, () => + expectedSession.window.close(), + ).pipe(Effect.ignore); + } }); - const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { - const recordingTabId = yield* Ref.get(recordingTabIdRef); - if (Option.isSome(recordingTabId) && recordingTabId.value !== tabId) { - return yield* new PreviewRecordingAlreadyActiveError({ - requestedTabId: tabId, - activeTabId: recordingTabId.value, + const closePictureInPictureUnlocked = Effect.fn("PreviewManager.closePictureInPictureUnlocked")( + function* (tabId: string) { + const pictureInPictureSession = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get( + tabId, + ); + if (!pictureInPictureSession) { + yield* stopFrameCapture(tabId, "picture-in-picture"); + const tabs = yield* SynchronizedRef.get(tabsRef); + if (tabs.has(tabId)) { + yield* update(tabId, { pictureInPicture: false }); + } + return; + } + yield* releasePictureInPicture(tabId, pictureInPictureSession, true); + }, + ); + + const closePictureInPicture = Effect.fn("PreviewManager.closePictureInPicture")(function* ( + tabId: string, + ) { + yield* pictureInPictureMutationSemaphore.withPermit(closePictureInPictureUnlocked(tabId)); + }); + + const closeAllPictureInPicture = Effect.fn("PreviewManager.closeAllPictureInPicture")( + function* () { + const sessions = yield* SynchronizedRef.get(pictureInPictureSessionsRef); + yield* Effect.forEach(sessions.keys(), closePictureInPicture, { + concurrency: "unbounded", + discard: true, }); + }, + ); + + const openPictureInPicture = Effect.fn("PreviewManager.openPictureInPicture")(function* ( + tabId: string, + ) { + const claim = yield* pictureInPictureMutationSemaphore.withPermit( + Effect.gen(function* () { + const existing = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (existing && !existing.window.isDestroyed()) { + return { kind: "existing" as const, session: existing }; + } + if (existing) { + yield* releasePictureInPicture(tabId, existing, false); + } + const wc = yield* requireWebContents(tabId); + const title = yield* attempt( + { + operation: "pictureInPicture.readTitle", + tabId, + webContentsId: wc.id, + }, + () => wc.getTitle().trim(), + ); + const pictureInPictureWindow = yield* attempt( + { + operation: "pictureInPicture.create", + tabId, + webContentsId: wc.id, + }, + () => + new BrowserWindow({ + width: PICTURE_IN_PICTURE_INITIAL_WIDTH, + height: PICTURE_IN_PICTURE_INITIAL_HEIGHT, + minWidth: PICTURE_IN_PICTURE_MIN_WIDTH, + minHeight: PICTURE_IN_PICTURE_MIN_HEIGHT, + title: title.length > 0 ? `Preview · ${title}` : "Browser preview", + show: false, + alwaysOnTop: true, + autoHideMenuBar: true, + fullscreenable: false, + maximizable: false, + minimizable: false, + resizable: true, + skipTaskbar: true, + backgroundColor: "#111111", + ...(hostPlatform === "darwin" ? { type: "panel" as const } : {}), + webPreferences: { + preload: pictureInPicturePreloadPath, + backgroundThrottling: false, + contextIsolation: true, + nodeIntegration: false, + sandbox: true, + }, + }), + ); + const initializationScope = yield* Scope.fork(parentScope, "sequential"); + const ready = yield* Deferred.make(); + const session: PictureInPictureSession = { + window: pictureInPictureWindow, + webContentsId: wc.id, + ready, + initializationScope, + }; + const onClosed = () => { + runFork( + pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, session, false), + ), + ); + }; + yield* attempt( + { + operation: "pictureInPicture.configure", + tabId, + webContentsId: wc.id, + }, + () => { + pictureInPictureWindow.once("closed", onClosed); + pictureInPictureWindow.setAlwaysOnTop( + true, + hostPlatform === "darwin" ? "floating" : "normal", + ); + if (hostPlatform === "darwin") { + pictureInPictureWindow.setVisibleOnAllWorkspaces(true, { + visibleOnFullScreen: true, + // Electron otherwise temporarily transforms the entire app into + // a UIElement process, which removes the owning app from the Dock. + skipTransformProcessType: true, + }); + } + }, + ).pipe( + Effect.onError(() => + Effect.all( + [ + Scope.close(initializationScope, Exit.void).pipe(Effect.ignore), + attempt({ operation: "pictureInPicture.close", tabId }, () => + pictureInPictureWindow.close(), + ).pipe(Effect.ignore), + ], + { discard: true }, + ), + ), + ); + yield* SynchronizedRef.update(pictureInPictureSessionsRef, (sessions) => + replaceMap(sessions, (copy) => { + copy.set(tabId, session); + }), + ); + return { kind: "created" as const, session }; + }), + ); + const pictureInPictureSession = claim.session; + if (claim.kind === "existing") { + yield* Deferred.await(pictureInPictureSession.ready); + return yield* pictureInPictureMutationSemaphore.withPermit( + Effect.gen(function* () { + const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (current !== pictureInPictureSession || pictureInPictureSession.window.isDestroyed()) { + return yield* new PreviewOperationError({ + operation: "pictureInPicture.showExisting", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + cause: new Error("Picture-in-picture session closed before it became visible."), + }); + } + yield* attempt( + { + operation: "pictureInPicture.showExisting", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + }, + () => pictureInPictureSession.window.showInactive(), + ); + }), + ); } - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "recording.start", startScreencast); - yield* Ref.set(recordingTabIdRef, Option.some(tabId)); + + const initialize = Effect.gen(function* () { + yield* attemptPromise( + { + operation: "pictureInPicture.load", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + }, + () => pictureInPictureSession.window.loadURL(buildPreviewPictureInPictureDataUrl()), + ); + const currentWebContents = yield* requireWebContents(tabId); + if ( + currentWebContents.id !== pictureInPictureSession.webContentsId || + currentWebContents.isDestroyed() + ) { + return yield* new PreviewOperationError({ + operation: "pictureInPicture.validateWebContents", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + cause: new Error("Preview webview changed while picture-in-picture was opening."), + }); + } + yield* startFrameCapture(tabId, "picture-in-picture"); + yield* attempt( + { + operation: "pictureInPicture.show", + tabId, + webContentsId: pictureInPictureSession.webContentsId, + }, + () => pictureInPictureSession.window.showInactive(), + ); + }); + const initializationExit = yield* Effect.gen(function* () { + const initializationFiber = yield* Effect.forkIn( + initialize, + pictureInPictureSession.initializationScope, + ); + return yield* Fiber.await(initializationFiber); + }).pipe( + Effect.onInterrupt(() => + pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, pictureInPictureSession, true), + ), + ), + ); + if (Exit.isSuccess(initializationExit)) { + const published = yield* pictureInPictureMutationSemaphore.withPermit( + Effect.gen(function* () { + const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (current !== pictureInPictureSession || pictureInPictureSession.window.isDestroyed()) { + if (current === pictureInPictureSession) { + yield* releasePictureInPicture(tabId, pictureInPictureSession, false); + } + return false; + } + yield* update(tabId, { pictureInPicture: true }); + yield* Deferred.done(pictureInPictureSession.ready, initializationExit); + return true; + }), + ); + if (published) return; + return yield* Deferred.await(pictureInPictureSession.ready); + } + yield* Deferred.done(pictureInPictureSession.ready, initializationExit); + const current = (yield* SynchronizedRef.get(pictureInPictureSessionsRef)).get(tabId); + if (current === pictureInPictureSession) { + yield* pictureInPictureMutationSemaphore.withPermit( + releasePictureInPicture(tabId, pictureInPictureSession, true), + ); + } + return yield* Effect.failCause(initializationExit.cause); + }); + + const startRecording = Effect.fn("PreviewManager.startRecording")(function* (tabId: string) { + yield* startFrameCapture(tabId, "recording"); }); const stopRecording = Effect.fn("PreviewManager.stopRecording")(function* (tabId: string) { - const recordingTabId = yield* Ref.get(recordingTabIdRef); - if (Option.isNone(recordingTabId) || recordingTabId.value !== tabId) return; - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "recording.stop", (send) => - send("Page.stopScreencast").pipe(Effect.asVoid), - ); - yield* Ref.set(recordingTabIdRef, Option.none()); + yield* stopFrameCapture(tabId, "recording"); }); const saveRecording = Effect.fn("PreviewManager.saveRecording")(function* ( @@ -2582,6 +3218,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function goForward, hardReload, navigate, + openPictureInPicture, openDevTools, pickElement, refresh, @@ -2593,6 +3230,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function setColorScheme, setMainWindow, startRecording, + closePictureInPicture, stopRecording, subscribePointerEvents: (listener: PointerEventListener) => subscribe(pointerEventListenersRef, listener), @@ -2678,18 +3316,6 @@ export class PreviewArtifactImageLoadError extends Schema.TaggedErrorClass()( - "PreviewRecordingAlreadyActiveError", - { - requestedTabId: Schema.String, - activeTabId: Schema.String, - }, -) { - override get message(): string { - return `Cannot record preview tab ${this.requestedTabId} while tab ${this.activeTabId} is already recording`; - } -} - export class PreviewAutomationDevToolsOpenError extends Schema.TaggedErrorClass()( "PreviewAutomationDevToolsOpenError", { webContentsId: Schema.Number }, @@ -2852,7 +3478,6 @@ export const PreviewManagerError = Schema.Union([ PreviewOperationError, PreviewArtifactPathOutsideDirectoryError, PreviewArtifactImageLoadError, - PreviewRecordingAlreadyActiveError, PreviewAutomationDevToolsOpenError, PreviewAutomationDebuggerAttachedError, PreviewAutomationEvaluationError, @@ -2915,6 +3540,8 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly revealArtifact: (path: string) => Effect.Effect; readonly copyArtifactToClipboard: (path: string) => Effect.Effect; + readonly openPictureInPicture: (tabId: string) => Effect.Effect; + readonly closePictureInPicture: (tabId: string) => Effect.Effect; readonly startRecording: (tabId: string) => Effect.Effect; readonly stopRecording: (tabId: string) => Effect.Effect; readonly saveRecording: ( @@ -2965,7 +3592,10 @@ export class PreviewManager extends Context.Service< export const make = Effect.gen(function* PreviewManagerMake() { const environment = yield* DesktopEnvironment.DesktopEnvironment; const browserSession = yield* BrowserSession.BrowserSession; - const operations = yield* makeNativeOperations(environment.browserArtifactsDir); + const operations = yield* makeNativeOperations( + environment.browserArtifactsDir, + environment.path.join(environment.dirname, "preview-pip-preload.cjs"), + ); return PreviewManager.of({ setMainWindow: operations.setMainWindow, @@ -3023,6 +3653,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { captureScreenshot: operations.captureScreenshot, revealArtifact: operations.revealArtifact, copyArtifactToClipboard: operations.copyArtifactToClipboard, + openPictureInPicture: operations.openPictureInPicture, + closePictureInPicture: operations.closePictureInPicture, startRecording: operations.startRecording, stopRecording: operations.stopRecording, saveRecording: operations.saveRecording, diff --git a/apps/desktop/src/window/DesktopWindow.test.ts b/apps/desktop/src/window/DesktopWindow.test.ts index 587da8d4431..3cd06cfcf32 100644 --- a/apps/desktop/src/window/DesktopWindow.test.ts +++ b/apps/desktop/src/window/DesktopWindow.test.ts @@ -315,6 +315,7 @@ const makeSplashScenario = (createOutcomes: readonly (Electron.BrowserWindow | n webPreferences: { preload: null, partition: null, + backgroundThrottling: null, sandbox: null, contextIsolation: null, nodeIntegration: null, @@ -429,6 +430,7 @@ describe("DesktopWindow", () => { assert.isUndefined(createdWindowOptions[0]?.x); assert.isUndefined(createdWindowOptions[0]?.y); assert.isTrue(createdWindowOptions[0]?.disableAutoHideCursor); + assert.isFalse(createdWindowOptions[0]?.webPreferences?.backgroundThrottling); assert.deepEqual(fakeWindow.setAutoHideCursor.mock.calls, [[false]]); assert.deepEqual(fakeWindow.loadURL.mock.calls[0], ["t3code-dev://app/"]); assert.equal(fakeWindow.openDevTools.mock.calls.length, 1); diff --git a/apps/desktop/src/window/DesktopWindow.ts b/apps/desktop/src/window/DesktopWindow.ts index db4b698434d..40788009d3b 100644 --- a/apps/desktop/src/window/DesktopWindow.ts +++ b/apps/desktop/src/window/DesktopWindow.ts @@ -331,6 +331,7 @@ export const make = Effect.gen(function* () { ...getWindowTitleBarOptions(shouldUseDarkColors, environment.platform), webPreferences: { preload: environment.preloadPath, + backgroundThrottling: false, contextIsolation: true, nodeIntegration: false, sandbox: true, diff --git a/apps/desktop/vite.config.ts b/apps/desktop/vite.config.ts index 96e089b9183..9f25204f163 100644 --- a/apps/desktop/vite.config.ts +++ b/apps/desktop/vite.config.ts @@ -73,5 +73,12 @@ export default defineConfig({ alwaysBundle: (id) => id === "react-grab" || id.startsWith("react-grab/"), }, }, + { + format: "cjs", + outDir: "dist-electron", + sourcemap: true, + outExtensions: () => ({ js: ".cjs" }), + entry: ["src/preview-pip-preload.ts"], + }, ], }); diff --git a/apps/server/src/auth/EnvironmentAuth.test.ts b/apps/server/src/auth/EnvironmentAuth.test.ts index 8432f49695a..440efcee51e 100644 --- a/apps/server/src/auth/EnvironmentAuth.test.ts +++ b/apps/server/src/auth/EnvironmentAuth.test.ts @@ -8,9 +8,9 @@ import * as ServerConfig from "../config.ts"; import { SqlitePersistenceMemory } from "../persistence/Layers/Sqlite.ts"; import * as PairingGrantStore from "./PairingGrantStore.ts"; import * as EnvironmentAuth from "./EnvironmentAuth.ts"; -import { resolveSessionCookieName } from "./utils.ts"; import * as ServerSecretStore from "./ServerSecretStore.ts"; +import * as SessionStore from "./SessionStore.ts"; /** Pinned so dev-mode cookie tests can assert the port-scoped name. */ const TEST_SERVER_PORT = 13_773; @@ -23,11 +23,8 @@ const makeServerConfigLayer = (overrides?: Partial while every request still sent t3_session_13773, - // and the tests would fail for a reason unrelated to what they assert. + // Keep the test server deterministic even when the default test layer + // changes its development port. port: TEST_SERVER_PORT, } satisfies ServerConfig.ServerConfig["Service"]; }), @@ -41,15 +38,12 @@ const makeEnvironmentAuthLayer = (overrides?: Partial[0] => ({ cookies: { - // Derived, not hardcoded: the name is port-scoped so concurrent servers - // on one hostname don't share a cookie. Mode and devUrl mirror - // ServerConfig.layerTest, so this resolves to whatever the server reads. - [resolveSessionCookieName({ mode: "web", port: TEST_SERVER_PORT, devUrl: undefined })]: - sessionToken, + [cookieName]: sessionToken, }, headers: {}, }) as unknown as Parameters< @@ -92,6 +86,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { it.effect("issues standard pairing credentials by default", () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; const pairingCredential = yield* serverAuth.issuePairingCredential(); const exchanged = yield* serverAuth.createBrowserSession( @@ -99,7 +94,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { requestMetadata, ); const verified = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(exchanged.sessionToken), + makeCookieRequest(sessions.cookieName, exchanged.sessionToken), ); expect(verified.sessionId.length).toBeGreaterThan(0); @@ -165,6 +160,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { it.effect("issues startup pairing URLs that bootstrap administrative sessions", () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; const pairingUrl = yield* serverAuth.issueStartupPairingUrl("http://127.0.0.1:3773"); const token = new URLSearchParams(new URL(pairingUrl).hash.slice(1)).get("token"); @@ -178,7 +174,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { const exchanged = yield* serverAuth.createBrowserSession(token ?? "", requestMetadata); const verified = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(exchanged.sessionToken), + makeCookieRequest(sessions.cookieName, exchanged.sessionToken), ); expect(verified.scopes).toEqual([ @@ -200,13 +196,14 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { () => Effect.gen(function* () { const serverAuth = yield* EnvironmentAuth.EnvironmentAuth; + const sessions = yield* SessionStore.SessionStore; const administrativeExchange = yield* serverAuth.createBrowserSession( "desktop-bootstrap-token", requestMetadata, ); const administrativeSession = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(administrativeExchange.sessionToken), + makeCookieRequest(sessions.cookieName, administrativeExchange.sessionToken), ); const pairingCredential = yield* serverAuth.issuePairingCredential({ label: "Julius iPhone", @@ -223,7 +220,7 @@ it.layer(NodeServices.layer)("EnvironmentAuth.layer", (it) => { }, ); const clientSession = yield* serverAuth.authenticateHttpRequest( - makeCookieRequest(clientExchange.sessionToken), + makeCookieRequest(sessions.cookieName, clientExchange.sessionToken), ); const clientsBeforeRevoke = yield* serverAuth.listClientSessions( administrativeSession.sessionId, diff --git a/apps/server/src/auth/EnvironmentAuthPolicy.test.ts b/apps/server/src/auth/EnvironmentAuthPolicy.test.ts index 0e21ef19c90..8e4c2171088 100644 --- a/apps/server/src/auth/EnvironmentAuthPolicy.test.ts +++ b/apps/server/src/auth/EnvironmentAuthPolicy.test.ts @@ -88,47 +88,50 @@ it.layer(NodeServices.layer)("EnvironmentAuthPolicy.layer", (it) => { expect(descriptor.policy).toBe("loopback-browser"); expect(descriptor.bootstrapMethods).toEqual(["one-time-token"]); - expect(descriptor.sessionCookieName).toBe("t3_session"); + expect(descriptor.sessionCookieName).toMatch(/^t3_session_3773_[a-f0-9]{12}$/); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ mode: "web", host: "127.0.0.1", - port: 13773, + port: 3773, }), ), ), ); - it.effect("scopes web session cookies by port only in development", () => + it.effect("uses remote-reachable policy for wildcard web hosts", () => Effect.gen(function* () { const policy = yield* EnvironmentAuthPolicy.EnvironmentAuthPolicy; const descriptor = yield* policy.getDescriptor(); - expect(descriptor.sessionCookieName).toBe("t3_session_13773"); + expect(descriptor.policy).toBe("remote-reachable"); + expect(descriptor.bootstrapMethods).toEqual(["one-time-token"]); + expect(descriptor.sessionCookieName).toBe("t3_session"); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ mode: "web", - port: 13773, - devUrl: new URL("http://127.0.0.1:5733"), + host: "0.0.0.0", }), ), ), ); - it.effect("uses remote-reachable policy for wildcard web hosts", () => + it.effect("isolates wildcard-bound web development sessions", () => Effect.gen(function* () { const policy = yield* EnvironmentAuthPolicy.EnvironmentAuthPolicy; const descriptor = yield* policy.getDescriptor(); expect(descriptor.policy).toBe("remote-reachable"); - expect(descriptor.bootstrapMethods).toEqual(["one-time-token"]); + expect(descriptor.sessionCookieName).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ mode: "web", host: "0.0.0.0", + port: 5775, + devUrl: new URL("http://127.0.0.1:5736"), }), ), ), @@ -140,6 +143,7 @@ it.layer(NodeServices.layer)("EnvironmentAuthPolicy.layer", (it) => { const descriptor = yield* policy.getDescriptor(); expect(descriptor.policy).toBe("remote-reachable"); + expect(descriptor.sessionCookieName).toBe("t3_session"); }).pipe( Effect.provide( makeEnvironmentAuthPolicyLayer({ diff --git a/apps/server/src/auth/EnvironmentAuthPolicy.ts b/apps/server/src/auth/EnvironmentAuthPolicy.ts index 28e41576769..9945c69067d 100644 --- a/apps/server/src/auth/EnvironmentAuthPolicy.ts +++ b/apps/server/src/auth/EnvironmentAuthPolicy.ts @@ -4,8 +4,7 @@ import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as ServerConfig from "../config.ts"; -import { resolveSessionCookieName } from "./utils.ts"; -import { isLoopbackHost, isWildcardHost } from "../startupAccess.ts"; +import { isRemoteReachableHost, resolveSessionCookieName } from "./utils.ts"; export class EnvironmentAuthPolicy extends Context.Service< EnvironmentAuthPolicy, @@ -16,7 +15,7 @@ export class EnvironmentAuthPolicy extends Context.Service< export const make = Effect.gen(function* () { const config = yield* ServerConfig.ServerConfig; - const isRemoteReachable = isWildcardHost(config.host) || !isLoopbackHost(config.host); + const isRemoteReachable = isRemoteReachableHost(config.host); const policy = config.mode === "desktop" @@ -41,7 +40,9 @@ export const make = Effect.gen(function* () { sessionCookieName: resolveSessionCookieName({ mode: config.mode, port: config.port, - devUrl: config.devUrl, + host: config.host, + instanceKey: config.stateDir, + development: config.devUrl !== undefined, }), }; diff --git a/apps/server/src/auth/SessionStore.ts b/apps/server/src/auth/SessionStore.ts index efa811302dc..40a1c43e0be 100644 --- a/apps/server/src/auth/SessionStore.ts +++ b/apps/server/src/auth/SessionStore.ts @@ -470,7 +470,9 @@ export const make = Effect.gen(function* () { const cookieName = resolveSessionCookieName({ mode: serverConfig.mode, port: serverConfig.port, - devUrl: serverConfig.devUrl, + host: serverConfig.host, + instanceKey: serverConfig.stateDir, + development: serverConfig.devUrl !== undefined, }); const emitUpsert = (clientSession: AuthClientSession) => diff --git a/apps/server/src/auth/utils.test.ts b/apps/server/src/auth/utils.test.ts index 90dc0f8ddf9..edc58f71131 100644 --- a/apps/server/src/auth/utils.test.ts +++ b/apps/server/src/auth/utils.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vite-plus/test"; -import { deriveAuthClientMetadata } from "./utils.ts"; +import { + deriveAuthClientMetadata, + isRemoteReachableHost, + resolveSessionCookieName, +} from "./utils.ts"; describe("deriveAuthClientMetadata", () => { it("labels Electron user agents as Electron instead of Chrome", () => { @@ -52,3 +56,80 @@ describe("deriveAuthClientMetadata", () => { expect(metadata.userAgent).toContain("Electron/36.3.2"); }); }); + +describe("session cookie isolation", () => { + it("isolates loopback web servers by port and server state", () => { + const first = resolveSessionCookieName({ + mode: "web", + port: 5775, + host: "127.0.0.1", + instanceKey: "/tmp/t3-agent-one", + development: true, + }); + const second = resolveSessionCookieName({ + mode: "web", + port: 5775, + host: "127.0.0.1", + instanceKey: "/tmp/t3-agent-two", + development: true, + }); + + expect(first).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + expect(second).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + expect(first).not.toBe(second); + }); + + it("keeps the hosted web cookie stable across server instances", () => { + expect( + resolveSessionCookieName({ + mode: "web", + port: 8080, + host: "0.0.0.0", + instanceKey: "/srv/release-a", + development: false, + }), + ).toBe("t3_session"); + expect( + resolveSessionCookieName({ + mode: "web", + port: 9090, + host: "app.example.com", + instanceKey: "/srv/release-b", + development: false, + }), + ).toBe("t3_session"); + }); + + it("retains desktop port scoping", () => { + expect( + resolveSessionCookieName({ + mode: "desktop", + port: 3773, + host: "127.0.0.1", + instanceKey: "/tmp/desktop", + development: true, + }), + ).toBe("t3_session_3773"); + }); + + it("isolates development servers even when they bind a wildcard host", () => { + expect( + resolveSessionCookieName({ + mode: "web", + port: 5775, + host: "0.0.0.0", + instanceKey: "/tmp/t3-wildcard-dev", + development: true, + }), + ).toMatch(/^t3_session_5775_[a-f0-9]{12}$/); + }); + + it("classifies loopback aliases separately from remotely reachable hosts", () => { + expect(isRemoteReachableHost(undefined)).toBe(false); + expect(isRemoteReachableHost("localhost")).toBe(false); + expect(isRemoteReachableHost("127.12.0.1")).toBe(false); + expect(isRemoteReachableHost("[::1]")).toBe(false); + expect(isRemoteReachableHost("0.0.0.0")).toBe(true); + expect(isRemoteReachableHost("192.168.1.50")).toBe(true); + }); +}); diff --git a/apps/server/src/auth/utils.ts b/apps/server/src/auth/utils.ts index 81ef9bffc49..32a6799b01f 100644 --- a/apps/server/src/auth/utils.ts +++ b/apps/server/src/auth/utils.ts @@ -28,11 +28,42 @@ const SESSION_COOKIE_NAME = "t3_session"; export function resolveSessionCookieName(input: { readonly mode: "web" | "desktop"; readonly port: number; - readonly devUrl: URL | undefined; + readonly host: string | undefined; + readonly instanceKey: string; + readonly development: boolean; }): string { - return input.devUrl === undefined && input.mode !== "desktop" - ? SESSION_COOKIE_NAME - : `${SESSION_COOKIE_NAME}_${input.port}`; + if (input.mode === "desktop") { + return `${SESSION_COOKIE_NAME}_${input.port}`; + } + + if (!input.development && isRemoteReachableHost(input.host)) { + return SESSION_COOKIE_NAME; + } + + // Cookies are scoped by host, not port. Loopback development servers need an + // instance-specific name or parallel agents overwrite each other's session, + // and a server that later reuses the port receives a token signed elsewhere. + const instanceHash = NodeCrypto.createHash("sha256") + .update(input.instanceKey) + .digest("hex") + .slice(0, 12); + return `${SESSION_COOKIE_NAME}_${input.port}_${instanceHash}`; +} + +export function isRemoteReachableHost(host: string | undefined): boolean { + if (host === "0.0.0.0" || host === "::" || host === "[::]") { + return true; + } + if (!host || host.length === 0) { + return false; + } + return !( + host === "localhost" || + host === "127.0.0.1" || + host === "::1" || + host === "[::1]" || + host.startsWith("127.") + ); } export function base64UrlEncode(input: string | Uint8Array): string { diff --git a/apps/server/src/mcp/PreviewAutomationBroker.test.ts b/apps/server/src/mcp/PreviewAutomationBroker.test.ts index ffda427d815..5bd665e5179 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.test.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.test.ts @@ -174,7 +174,7 @@ it.effect("does not let an older response replace a newer explicit tab target", ), ); -it.effect("does not replace the default tab with a globally stopped recording tab", () => +it.effect("tracks the tab returned by a targeted recording stop", () => Effect.scoped( Effect.gen(function* () { const broker = yield* makeBroker; @@ -203,7 +203,7 @@ it.effect("does not replace the default tab with a globally stopped recording ta yield* broker.invoke({ scope, operation: "recordingStop", input: {} }); yield* broker.invoke({ scope, operation: "snapshot", input: {} }); - expect(routedRequests.at(-1)?.tabId).toBe(browsingTabId); + expect(routedRequests.at(-1)?.tabId).toBe(recordingTabId); }), ), ); diff --git a/apps/server/src/mcp/PreviewAutomationBroker.ts b/apps/server/src/mcp/PreviewAutomationBroker.ts index 7ed77aabdf1..5a952803033 100644 --- a/apps/server/src/mcp/PreviewAutomationBroker.ts +++ b/apps/server/src/mcp/PreviewAutomationBroker.ts @@ -548,8 +548,7 @@ export const make = Effect.gen(function* PreviewAutomationBrokerMake() { }); }); const result = yield* awaitResponse().pipe(Effect.ensuring(removePending)); - // A stop artifact identifies the globally recorded tab, not the caller's browsing target. - const responseTabId = input.operation === "recordingStop" ? undefined : readResultTabId(result); + const responseTabId = readResultTabId(result); const resultTabId = responseTabId === undefined ? input.tabId : responseTabId; if (resultTabId === undefined) return result; const assignmentKey = hostAssignmentKey(input.scope); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.test.ts b/apps/server/src/mcp/toolkits/preview/handlers.test.ts new file mode 100644 index 00000000000..93985fc9d4c --- /dev/null +++ b/apps/server/src/mcp/toolkits/preview/handlers.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { normalizePreviewOpenInput } from "./handlers.ts"; + +describe("normalizePreviewOpenInput", () => { + it("opens the inline preview and reuses the current tab by default", () => { + expect(normalizePreviewOpenInput({})).toEqual({ + open: true, + reuseExistingTab: true, + show: true, + }); + }); + + it("preserves an explicit background-only opt-out", () => { + expect(normalizePreviewOpenInput({ open: false })).toEqual({ + open: false, + reuseExistingTab: true, + show: false, + }); + }); + + it("supports show as a legacy alias while preferring open", () => { + expect(normalizePreviewOpenInput({ show: false })).toEqual({ + open: false, + reuseExistingTab: true, + show: false, + }); + expect(normalizePreviewOpenInput({ open: true, show: false })).toEqual({ + open: true, + reuseExistingTab: true, + show: true, + }); + }); +}); diff --git a/apps/server/src/mcp/toolkits/preview/handlers.ts b/apps/server/src/mcp/toolkits/preview/handlers.ts index 8c4651dc1cf..1c7ff6f9cd9 100644 --- a/apps/server/src/mcp/toolkits/preview/handlers.ts +++ b/apps/server/src/mcp/toolkits/preview/handlers.ts @@ -1,6 +1,7 @@ import * as Effect from "effect/Effect"; import type { PreviewAutomationOperation, + PreviewAutomationOpenInput, PreviewAutomationRecordingArtifact, PreviewAutomationRecordingStatus, PreviewAutomationResizeResult, @@ -14,6 +15,18 @@ import * as McpInvocationContext from "../../McpInvocationContext.ts"; import * as PreviewAutomationBroker from "../../PreviewAutomationBroker.ts"; import { PreviewSnapshotToolkit, PreviewStandardToolkit, PreviewToolkit } from "./tools.ts"; +export function normalizePreviewOpenInput( + input: PreviewAutomationOpenInput, +): PreviewAutomationOpenInput { + const open = input.open ?? input.show ?? true; + return { + ...input, + open, + show: open, + reuseExistingTab: input.reuseExistingTab ?? true, + }; +} + const invoke = Effect.fn("PreviewToolkit.invoke")(function* ( operation: PreviewAutomationOperation, input: unknown, @@ -50,11 +63,7 @@ const invokeTargeted = ( const handlers = { preview_status: (input) => invokeTargeted("status", input ?? {}), preview_open: (input) => - invokeTargeted("open", { - ...input, - show: input.show ?? true, - reuseExistingTab: input.reuseExistingTab ?? true, - }), + invokeTargeted("open", normalizePreviewOpenInput(input)), preview_navigate: (input) => invokeTargeted("navigate", input, input.timeoutMs), preview_resize: (input) => diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index d2527fdfb39..a94d2b056f7 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -54,7 +54,7 @@ export const PreviewStatusTool = Tool.make("preview_status", { export const PreviewOpenTool = browserTool( Tool.make("preview_open", { description: - "Show and initialize a collaborative browser tab. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", + "Initialize a collaborative browser tab and open its thread-bound inline preview by default. Set open=false for background-only automation. Pass tabId to reuse a specific existing tab, set reuseExistingTab=false to create another tab, or omit both to use this agent session's current tab.", parameters: PreviewAutomationOpenInput, success: PreviewAutomationStatus, failure: PreviewAutomationError, @@ -191,7 +191,8 @@ export const PreviewRecordingStartTool = safeBrowserTool( export const PreviewRecordingStopTool = safeBrowserTool( Tool.make("preview_recording_stop", { - description: "Stop the active browser recording and save it as a local evidence artifact.", + description: + "Stop recording the collaborative browser tab selected by tabId, or this agent session's current tab when omitted, and save it as a local evidence artifact.", parameters: PreviewAutomationTabTargetInput, success: PreviewAutomationRecordingArtifact, failure: PreviewAutomationError, diff --git a/apps/server/src/preview/Manager.test.ts b/apps/server/src/preview/Manager.test.ts index 693111b578f..8b3dabfa338 100644 --- a/apps/server/src/preview/Manager.test.ts +++ b/apps/server/src/preview/Manager.test.ts @@ -67,6 +67,32 @@ it.layer(PreviewManager.layer)("PreviewManager", (it) => { }), ); + it.effect("orders list snapshots and events with one monotonic revision", () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + const collector = yield* collectEvents; + const before = yield* manager.list({ threadId }); + + const opened = yield* manager.open({ threadId, url: "http://localhost:5173" }); + yield* manager.navigate({ + threadId, + tabId: opened.tabId, + url: "http://localhost:5173/ready", + }); + + const events = yield* collector.drain; + const listed = yield* manager.list({ threadId }); + expect(events).toHaveLength(2); + expect(events[0]!.serverEpoch).toBe(listed.serverEpoch); + expect(events[1]!.serverEpoch).toBe(listed.serverEpoch); + expect(events[0]!.revision).toBeGreaterThan(before.revision); + expect(events[1]!.revision).toBeGreaterThan(events[0]!.revision); + expect(listed.revision).toBe(events[1]!.revision); + expect(listed.sessions).toHaveLength(1); + }), + ); + it.effect("treats bare hosts as https", () => Effect.gen(function* () { const threadId = freshThreadId(); @@ -254,6 +280,26 @@ it.layer(PreviewManager.layer)("PreviewManager", (it) => { }), ); + it.effect("gives every tab in a batch close its own monotonic revision", () => + Effect.gen(function* () { + const threadId = freshThreadId(); + const manager = yield* PreviewManager.PreviewManager; + yield* manager.open({ threadId, url: "http://localhost:5173" }); + yield* manager.open({ threadId, url: "http://localhost:3000" }); + const collector = yield* collectEvents; + + yield* manager.close({ threadId }); + + const events = yield* collector.drain; + const listed = yield* manager.list({ threadId }); + expect(events).toHaveLength(2); + expect(events.every((event) => event.type === "closed")).toBe(true); + expect(events[1]!.revision).toBeGreaterThan(events[0]!.revision); + expect(listed.revision).toBe(events[1]!.revision); + expect(listed.sessions).toHaveLength(0); + }), + ); + it.effect("close is idempotent for unknown threads", () => Effect.gen(function* () { const threadId = freshThreadId(); diff --git a/apps/server/src/preview/Manager.ts b/apps/server/src/preview/Manager.ts index 193ea85b21b..e38e28ecd2e 100644 --- a/apps/server/src/preview/Manager.ts +++ b/apps/server/src/preview/Manager.ts @@ -30,6 +30,7 @@ import { newPreviewTabId, normalizePreviewUrl, } from "@t3tools/shared/preview"; +import * as NodeCrypto from "node:crypto"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -67,9 +68,17 @@ interface PreviewSessionState { interface ManagerState { /** All sessions across every thread, keyed by `${threadId}\u0000${tabId}`. */ readonly sessions: ReadonlyMap; + /** Global monotonic revision establishing list/event ordering. */ + readonly revision: number; } -const initialState: ManagerState = { sessions: new Map() }; +const initialState: ManagerState = { sessions: new Map(), revision: 0 }; + +type PreviewEventDraft = PreviewEvent extends infer Event + ? Event extends { readonly revision: number } + ? Omit + : never + : never; const compositeKey = (threadId: string, tabId: string): string => `${threadId}\u0000${tabId}`; @@ -138,6 +147,7 @@ const buildIdleSnapshot = (input: { }); export const make = Effect.gen(function* PreviewManagerMake() { + const serverEpoch = NodeCrypto.randomUUID(); const stateRef = yield* SynchronizedRef.make(initialState); // Unbounded PubSub is fine here — events are tiny and we don't want to // block publishers if a subscriber is slow. WS clients backpressure on @@ -160,7 +170,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { tabId: string, mutator: ( session: PreviewSessionState, - ) => Effect.Effect<{ next: PreviewSessionState; emit: PreviewEvent | null; result: R }, E>, + ) => Effect.Effect<{ next: PreviewSessionState; emit: PreviewEventDraft | null; result: R }, E>, ): Effect.Effect => { type ModifyResult = | { kind: "fail"; error: PreviewSessionLookupError } @@ -177,10 +187,17 @@ export const make = Effect.gen(function* PreviewManagerMake() { return mutator(session).pipe( Effect.flatMap( Effect.fn("PreviewManager.commitMutation")(function* ({ next, emit, result }) { - if (emit) yield* PubSub.publish(eventsPubSub, emit); + const revision = emit ? state.revision + 1 : state.revision; + if (emit) { + yield* PubSub.publish(eventsPubSub, { + ...emit, + revision, + serverEpoch, + } as PreviewEvent); + } const sessions = new Map(state.sessions); sessions.set(compositeKey(threadId, tabId), next); - return [{ kind: "ok", result } as ModifyResult, { sessions }] as readonly [ + return [{ kind: "ok", result } as ModifyResult, { sessions, revision }] as readonly [ ModifyResult, ManagerState, ]; @@ -207,22 +224,27 @@ export const make = Effect.gen(function* PreviewManagerMake() { updatedAt, }) : buildIdleSnapshot({ threadId: input.threadId, tabId, updatedAt }); - yield* SynchronizedRef.update(stateRef, (state) => { - const sessions = new Map(state.sessions); - sessions.set(compositeKey(input.threadId, tabId), { - threadId: input.threadId, - tabId, - snapshot, - }); - return { sessions }; - }); - yield* PubSub.publish(eventsPubSub, { - type: "opened", - threadId: input.threadId, - tabId, - createdAt: snapshot.updatedAt, - snapshot, - }); + yield* SynchronizedRef.modifyEffect(stateRef, (state) => + Effect.gen(function* () { + const revision = state.revision + 1; + const sessions = new Map(state.sessions); + sessions.set(compositeKey(input.threadId, tabId), { + threadId: input.threadId, + tabId, + snapshot, + }); + yield* PubSub.publish(eventsPubSub, { + type: "opened", + threadId: input.threadId, + tabId, + createdAt: snapshot.updatedAt, + serverEpoch, + revision, + snapshot, + }); + return [snapshot, { sessions, revision }] as const; + }), + ); return snapshot; }, ); @@ -280,7 +302,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { viewport: session.snapshot.viewport ?? FILL_PREVIEW_VIEWPORT, updatedAt, }; - const emit: PreviewEvent = + const emit: PreviewEventDraft = input.navStatus._tag === "LoadFailed" ? { type: "failed", @@ -349,7 +371,7 @@ export const make = Effect.gen(function* PreviewManagerMake() { const close: PreviewManager["Service"]["close"] = Effect.fn("PreviewManager.close")( function* (input) { const createdAt = yield* currentIsoTimestamp; - const events = yield* SynchronizedRef.modify(stateRef, (state) => { + yield* SynchronizedRef.modifyEffect(stateRef, (state) => { const eventsToEmit: PreviewEvent[] = []; const sessions = new Map(state.sessions); const targets = input.tabId @@ -357,25 +379,29 @@ export const make = Effect.gen(function* PreviewManagerMake() { (entry): entry is PreviewSessionState => entry !== undefined, ) : sessionsForThread(state, input.threadId); + let revision = state.revision; for (const target of targets) { + revision += 1; sessions.delete(compositeKey(target.threadId, target.tabId)); eventsToEmit.push({ type: "closed", threadId: target.threadId, tabId: target.tabId, createdAt, + serverEpoch, + revision, }); } if (eventsToEmit.length === 0) { - return [eventsToEmit, state] as const; + return Effect.succeed([undefined, state] as const); } - return [eventsToEmit, { sessions }] as const; + return Effect.as( + Effect.forEach(eventsToEmit, (event) => PubSub.publish(eventsPubSub, event), { + discard: true, + }), + [undefined, { sessions, revision }] as const, + ); }); - if (events.length > 0) { - yield* Effect.forEach(events, (event) => PubSub.publish(eventsPubSub, event), { - discard: true, - }); - } }, ); @@ -387,6 +413,8 @@ export const make = Effect.gen(function* PreviewManagerMake() { sessions: sessionsForThread(state, input.threadId) .map((s) => s.snapshot) .toSorted((a, b) => a.updatedAt.localeCompare(b.updatedAt)), + serverEpoch, + revision: state.revision, }), ), ); diff --git a/apps/server/src/server.test.ts b/apps/server/src/server.test.ts index f06c27a066e..4a9e7306217 100644 --- a/apps/server/src/server.test.ts +++ b/apps/server/src/server.test.ts @@ -667,7 +667,7 @@ const buildAppUnderTest = (options?: { reportStatus: () => Effect.void, refresh: () => Effect.void, close: () => Effect.void, - list: () => Effect.succeed({ sessions: [] }), + list: () => Effect.succeed({ sessions: [], serverEpoch: "test-server", revision: 0 }), events: Stream.empty, subscribeEvents: Effect.flatMap(PubSub.unbounded(), (pubsub) => PubSub.subscribe(pubsub), diff --git a/apps/web/src/browser/BrowserSurfaceSlot.tsx b/apps/web/src/browser/BrowserSurfaceSlot.tsx index de74cfa2a90..a9d3f541ff1 100644 --- a/apps/web/src/browser/BrowserSurfaceSlot.tsx +++ b/apps/web/src/browser/BrowserSurfaceSlot.tsx @@ -7,27 +7,56 @@ import { acquireBrowserSurface } from "./browserSurfaceStore"; export function BrowserSurfaceSlot(props: { readonly tabId: string; readonly visible: boolean; + readonly cornerRadius?: number; + readonly layoutVersion?: string | number; readonly className?: string; + readonly fitSourceContent?: boolean; }) { - const { tabId, visible, className } = props; + const { + tabId, + visible, + cornerRadius = 0, + layoutVersion, + className, + fitSourceContent = false, + } = props; const elementRef = useRef(null); + const presentationRef = useRef({ visible, cornerRadius }); + const updateRef = useRef<(() => void) | null>(null); useLayoutEffect(() => { const element = elementRef.current; if (!element) return; - const lease = acquireBrowserSurface(tabId); + let lease = acquireBrowserSurface(tabId, fitSourceContent); const update = () => { const rect = element.getBoundingClientRect(); - lease.present( + const presentation = presentationRef.current; + const presented = lease.present( { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.max(1, Math.round(rect.width)), height: Math.max(1, Math.round(rect.height)), }, - visible && rect.width > 0 && rect.height > 0, + presentation.visible && rect.width > 0 && rect.height > 0, + presentation.cornerRadius, ); + if (presentation.visible && !presented) { + lease.release(); + lease = acquireBrowserSurface(tabId, fitSourceContent); + lease.present( + { + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.max(1, Math.round(rect.width)), + height: Math.max(1, Math.round(rect.height)), + }, + rect.width > 0 && rect.height > 0, + presentation.cornerRadius, + ); + } }; + updateRef.current = update; update(); const observer = new ResizeObserver(update); observer.observe(element); @@ -37,9 +66,15 @@ export function BrowserSurfaceSlot(props: { observer.disconnect(); window.removeEventListener("resize", update); window.removeEventListener("scroll", update, true); + if (updateRef.current === update) updateRef.current = null; lease.release(); }; - }, [tabId, visible]); + }, [fitSourceContent, tabId]); + + useLayoutEffect(() => { + presentationRef.current = { visible, cornerRadius }; + updateRef.current?.(); + }, [cornerRadius, layoutVersion, visible]); return
; } diff --git a/apps/web/src/browser/HostedBrowserWebview.tsx b/apps/web/src/browser/HostedBrowserWebview.tsx index a72da49c6fa..1d8b1e7db07 100644 --- a/apps/web/src/browser/HostedBrowserWebview.tsx +++ b/apps/web/src/browser/HostedBrowserWebview.tsx @@ -8,15 +8,19 @@ import { previewBridge } from "~/components/preview/previewBridge"; import { usePreviewBridge } from "~/components/preview/usePreviewBridge"; import { cn } from "~/lib/utils"; -import { stopBrowserRecording, useActiveBrowserRecordingTabId } from "./browserRecording"; import { resolveBrowserSurfacePanelRect, useBrowserSurfaceStore } from "./browserSurfaceStore"; -import { browserViewportSettingKey } from "./browserViewportLayout"; +import { browserViewportSettingKey, resolveBrowserViewportLayout } from "./browserViewportLayout"; import { BrowserDeviceToolbar } from "./BrowserDeviceToolbar"; import { BrowserViewportResizeHandles } from "./BrowserViewportResizeHandles"; import { acquireDesktopTab, type AcquiredDesktopTab } from "./desktopTabLifetime"; import { resolveHostedBrowserWebviewWrapperStyle } from "./hostedBrowserWebviewStyle"; import { usePreviewWebviewConfig } from "./previewWebviewConfigState"; import { useBrowserViewportResize } from "./useBrowserViewportResize"; +import { + INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, + planWebviewCrashRecovery, + type WebviewCrashRecoveryState, +} from "./webviewCrashRecovery"; interface ElectronWebview extends HTMLElement { src: string; @@ -46,12 +50,15 @@ export function HostedBrowserWebview(props: { const tabLeaseRef = useRef(null); const wrapperRef = useRef(null); const webviewRef = useRef(null); + const crashRecoveryRef = useRef(INITIAL_WEBVIEW_CRASH_RECOVERY_STATE); const [aspectRatioLocked, setAspectRatioLocked] = useState(false); - const activeRecordingTabId = useActiveBrowserRecordingTabId(); const presentation = useBrowserSurfaceStore( useShallow((state) => { const current = state.byTabId[tabId]; return { + cornerRadius: current?.cornerRadius ?? 0, + fitSourceContent: current?.fitSourceContent ?? false, + fittedSourceContent: current?.fittedSourceContent ?? null, rect: resolveBrowserSurfacePanelRect(state.byTabId, tabId), visible: current?.visible ?? false, }; @@ -60,11 +67,7 @@ export function HostedBrowserWebview(props: { usePreviewBridge({ threadRef, tabId }); useEffect(() => { - if (presentation.visible || activeRecordingTabId !== tabId) return; - void stopBrowserRecording(tabId).catch(() => undefined); - }, [activeRecordingTabId, presentation.visible, tabId]); - - useEffect(() => { + crashRecoveryRef.current = INITIAL_WEBVIEW_CRASH_RECOVERY_STATE; const lease = acquireDesktopTab(tabId); tabLeaseRef.current = lease; return () => { @@ -73,6 +76,14 @@ export function HostedBrowserWebview(props: { }; }, [tabId]); + const [webviewGeneration, setWebviewGeneration] = useState(0); + const [recoverySrc, setRecoverySrc] = useState(initialSrc); + const latestUrlRef = useRef(initialUrl); + + useEffect(() => { + latestUrlRef.current = initialUrl; + }, [initialUrl]); + const setWebviewRef = useCallback((node: HTMLElement | null) => { webviewRef.current = node as ElectronWebview | null; if (node && !node.hasAttribute("allowpopups")) node.setAttribute("allowpopups", "true"); @@ -83,6 +94,7 @@ export function HostedBrowserWebview(props: { const bridge = previewBridge; if (!webview || !config || !bridge) return; let disposed = false; + let recoveryTimeout: ReturnType | null = null; const register = () => { const lease = tabLeaseRef.current; if (!lease) return; @@ -102,15 +114,31 @@ export function HostedBrowserWebview(props: { } })(); }; + const recoverGuest = () => { + if (disposed || recoveryTimeout !== null) return; + const recovery = planWebviewCrashRecovery(crashRecoveryRef.current, Date.now()); + if (!recovery) return; + crashRecoveryRef.current = recovery.state; + recoveryTimeout = setTimeout(() => { + recoveryTimeout = null; + if (!disposed) { + setRecoverySrc(latestUrlRef.current ?? initialSrc); + setWebviewGeneration((generation) => generation + 1); + } + }, recovery.delayMs); + }; webview.addEventListener("did-attach", register); webview.addEventListener("dom-ready", register); + webview.addEventListener("render-process-gone", recoverGuest); register(); return () => { disposed = true; + if (recoveryTimeout !== null) clearTimeout(recoveryTimeout); webview.removeEventListener("did-attach", register); webview.removeEventListener("dom-ready", register); + webview.removeEventListener("render-process-gone", recoverGuest); }; - }, [config, tabId]); + }, [config, initialSrc, tabId, webviewGeneration]); const active = presentation.visible && presentation.rect !== null; const lastRect = presentation.rect; @@ -132,14 +160,14 @@ export function HostedBrowserWebview(props: { } : { width: lastRect?.width ?? 1280, height: lastRect?.height ?? 800 }; const containerSize = active && lastRect ? lastRect : hiddenSize; - const deviceToolbarVisible = active && viewport._tag !== "fill"; + const deviceToolbarVisible = active && viewport._tag !== "fill" && !presentation.fitSourceContent; const { activeDrag, commitViewportChange, effectiveViewport, handleResizeKeyDown, handleResizePointerDown, - layout, + layout: viewportLayout, } = useBrowserViewportResize({ tabId, viewport, @@ -148,6 +176,38 @@ export function HostedBrowserWebview(props: { deviceToolbarVisible, aspectRatio: lockedAspectRatio, }); + const fittedSourceViewport = + presentation.fitSourceContent && lastRect + ? presentation.fittedSourceContent + ? { + _tag: "freeform" as const, + width: Math.max( + 1, + Math.round( + presentation.fittedSourceContent.width / + presentation.fittedSourceContent.scale / + normalizedZoomFactor, + ), + ), + height: Math.max( + 1, + Math.round( + presentation.fittedSourceContent.height / + presentation.fittedSourceContent.scale / + normalizedZoomFactor, + ), + ), + } + : { + _tag: "freeform" as const, + width: viewport._tag === "fill" ? 1280 : viewport.width, + height: viewport._tag === "fill" ? 800 : viewport.height, + } + : null; + const layout = + fittedSourceViewport && lastRect + ? resolveBrowserViewportLayout(lastRect, fittedSourceViewport, normalizedZoomFactor) + : viewportLayout; const syncContentPresentation = useCallback(() => { const wrapper = wrapperRef.current; @@ -178,6 +238,7 @@ export function HostedBrowserWebview(props: { const wrapperStyle = resolveHostedBrowserWebviewWrapperStyle({ active, + cornerRadius: presentation.cornerRadius, rect: lastRect, hiddenSize, }); @@ -201,8 +262,9 @@ export function HostedBrowserWebview(props: { /> ) : null} - {active && effectiveViewport._tag !== "fill" ? ( + {active && effectiveViewport._tag !== "fill" && !fittedSourceViewport ? ( <> { - const events: string[] = []; - const surfaceState = { - byTabId: {} as Record, - }; - return { - events, - onFrame: vi.fn(() => vi.fn()), - registrySet: vi.fn((_atom: unknown, value: string | null) => { - events.push(value === null ? "clear" : `publish:${value}`); - }), - save: vi.fn(async () => ({ - id: "recording-test", - tabId: "recording-tab", - path: "/tmp/recording-test.webm", - mimeType: "video/webm" as const, - sizeBytes: 0, - createdAt: "2026-06-26T00:00:00.000Z", - })), - startScreencast: vi.fn(async () => { - events.push("start-screencast"); - }), - stopScreencast: vi.fn(async () => undefined), - surfaceState, - }; - }); +const { + events, + frameSubscription, + onFrame, + registrySet, + save, + startScreencast, + stopScreencast, + surfaceState, +} = vi.hoisted(() => { + const events: string[] = []; + type Frame = { + readonly tabId: string; + readonly data: string; + readonly width: number; + readonly height: number; + readonly receivedAt: string; + }; + const frameSubscription: { listener: ((frame: Frame) => void) | null } = { + listener: null, + }; + const surfaceState = { + byTabId: {} as Record, + }; + return { + events, + frameSubscription, + onFrame: vi.fn((listener: (frame: Frame) => void) => { + frameSubscription.listener = listener; + return () => { + if (frameSubscription.listener === listener) frameSubscription.listener = null; + }; + }), + registrySet: vi.fn((_atom: unknown, value: { readonly tabIds: ReadonlySet }) => { + events.push( + value.tabIds.size === 0 ? "clear" : `publish:${Array.from(value.tabIds).join(",")}`, + ); + }), + save: vi.fn(async (tabId: string) => ({ + id: "recording-test", + tabId, + path: "/tmp/recording-test.webm", + mimeType: "video/webm" as const, + sizeBytes: 0, + createdAt: "2026-06-26T00:00:00.000Z", + })), + startScreencast: vi.fn(async (tabId: string) => { + events.push("start-screencast"); + const surface = surfaceState.byTabId[tabId] as + | { + readonly content?: { readonly width: number; readonly height: number }; + readonly rect?: { readonly width: number; readonly height: number }; + } + | undefined; + const size = surface?.content ?? surface?.rect; + frameSubscription.listener?.({ + tabId, + data: "initial-frame", + width: size?.width ?? 1280, + height: size?.height ?? 800, + receivedAt: "2026-06-26T00:00:00.000Z", + }); + }), + stopScreencast: vi.fn(async () => undefined), + surfaceState, + }; +}); vi.mock("~/components/preview/previewBridge", () => ({ previewBridge: { @@ -45,10 +86,10 @@ vi.mock("./browserSurfaceStore", () => ({ })); import { + BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS, BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS, BrowserRecordingConflictError, - BrowserRecordingOperationError, - BrowserRecordingRequiresVisibleTabError, + readActiveBrowserRecordingTabIds, startBrowserRecording, stopBrowserRecording, } from "./browserRecording"; @@ -80,9 +121,20 @@ class FakeMediaRecorder { } } +const emitRecordingFrame = () => { + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "startup-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:00.000Z", + }); +}; + describe("browser recording", () => { beforeEach(() => { events.length = 0; + frameSubscription.listener = null; surfaceState.byTabId = { "recording-tab": { visible: true, @@ -93,12 +145,26 @@ describe("browser recording", () => { vi.clearAllMocks(); vi.stubGlobal("window", globalThis); vi.stubGlobal("MediaRecorder", FakeMediaRecorder as unknown as typeof MediaRecorder); + class ImmediateImage { + private loadListener: EventListenerOrEventListenerObject | undefined; + + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { + if (type === "load") this.loadListener = listener; + } + + set src(_value: string) { + const event = new Event("load"); + if (typeof this.loadListener === "function") this.loadListener(event); + else this.loadListener?.handleEvent(event); + } + } + vi.stubGlobal("Image", ImmediateImage as unknown as typeof Image); vi.stubGlobal("document", { createElement: () => ({ width: 0, height: 0, captureStream: () => ({}), - getContext: () => ({ drawImage: vi.fn() }), + getContext: () => ({ drawImage: vi.fn(), fillRect: vi.fn(), fillStyle: "" }), }), }); }); @@ -116,7 +182,7 @@ describe("browser recording", () => { await stopBrowserRecording("recording-tab"); }); - it("rejects recording for a hidden tab before starting screencast", async () => { + it("records a hidden tab without requiring it to become visible", async () => { surfaceState.byTabId = { "recording-tab": { visible: false, @@ -125,18 +191,189 @@ describe("browser recording", () => { }, }; - await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf( - BrowserRecordingRequiresVisibleTabError, + await startBrowserRecording("recording-tab"); + + expect(startScreencast).toHaveBeenCalledWith("recording-tab"); + expect(events).toEqual(["start-screencast", "publish:recording-tab"]); + + await stopBrowserRecording("recording-tab"); + }); + + it("fails startup instead of locking a fallback size when no frame arrives", async () => { + vi.useFakeTimers(); + startScreencast.mockImplementationOnce(async () => { + events.push("start-screencast"); + }); + + const startPromise = startBrowserRecording("recording-tab"); + const rejection = expect(startPromise).rejects.toMatchObject({ + operation: "wait-first-frame", + tabId: "recording-tab", + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); + + await rejection; + expect(stopScreencast).toHaveBeenCalledWith("recording-tab"); + expect(events.at(-1)).toBe("clear"); + }); + + it("fixes hidden recording dimensions before MediaRecorder starts", async () => { + const drawImage = vi.fn(); + const fillRect = vi.fn(); + let capturedStreamSize: { readonly width: number; readonly height: number } | undefined; + const canvas = { + width: 0, + height: 0, + captureStream: () => { + capturedStreamSize = { width: canvas.width, height: canvas.height }; + return {}; + }, + getContext: () => ({ drawImage, fillRect, fillStyle: "" }), + }; + vi.stubGlobal("document", { + createElement: () => canvas, + }); + surfaceState.byTabId = {}; + startScreencast.mockImplementationOnce(async (tabId: string) => { + events.push("start-screencast"); + frameSubscription.listener?.({ + tabId, + data: "captured-frame", + width: 390, + height: 844, + receivedAt: "2026-06-26T00:00:00.000Z", + }); + }); + + await startBrowserRecording("recording-tab"); + + expect(canvas).toMatchObject({ width: 390, height: 844 }); + expect(capturedStreamSize).toEqual({ width: 390, height: 844 }); + expect(drawImage).toHaveBeenCalledWith(expect.anything(), 0, 0, 390, 844); + + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "different-sized-frame", + width: 1280, + height: 720, + receivedAt: "2026-06-26T00:00:01.000Z", + }); + + expect(canvas).toMatchObject({ width: 390, height: 844 }); + expect(fillRect).toHaveBeenLastCalledWith(0, 0, 390, 844); + + await stopBrowserRecording("recording-tab"); + }); + + it("draws the newest decoded frames without starving behind decode latency", async () => { + const drawImage = vi.fn(); + class DeferredImage { + static readonly instances: DeferredImage[] = []; + private loadListener: EventListenerOrEventListenerObject | undefined; + + constructor() { + DeferredImage.instances.push(this); + } + + addEventListener(type: string, listener: EventListenerOrEventListenerObject): void { + if (type === "load") this.loadListener = listener; + } + + set src(_value: string) {} + + finishLoading(): void { + const event = new Event("load"); + if (typeof this.loadListener === "function") this.loadListener(event); + else this.loadListener?.handleEvent(event); + } + } + vi.stubGlobal("Image", DeferredImage as unknown as typeof Image); + vi.stubGlobal("document", { + createElement: () => ({ + width: 0, + height: 0, + captureStream: () => ({}), + getContext: () => ({ drawImage, fillRect: vi.fn(), fillStyle: "" }), + }), + }); + + await startBrowserRecording("recording-tab"); + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "second-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:01.000Z", + }); + frameSubscription.listener?.({ + tabId: "recording-tab", + data: "third-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:02.000Z", + }); + + DeferredImage.instances[1]?.finishLoading(); + expect(drawImage).toHaveBeenCalledOnce(); + DeferredImage.instances[2]?.finishLoading(); + expect(drawImage).toHaveBeenCalledTimes(2); + DeferredImage.instances[0]?.finishLoading(); + expect(drawImage).toHaveBeenCalledTimes(2); + + await stopBrowserRecording("recording-tab"); + }); + + it("records separate tabs concurrently", async () => { + const firstThreadRef = { + environmentId: EnvironmentId.make("environment-recording"), + threadId: ThreadId.make("thread-recording-first"), + }; + const secondThreadRef = { + environmentId: EnvironmentId.make("environment-recording"), + threadId: ThreadId.make("thread-recording-second"), + }; + surfaceState.byTabId = { + ...surfaceState.byTabId, + "recording-tab-2": { + visible: false, + rect: { x: 0, y: 0, width: 390, height: 844 }, + content: { x: 0, y: 0, width: 390, height: 844, scale: 1, scrollLeft: 0, scrollTop: 0 }, + }, + }; + + await Promise.all([ + startBrowserRecording("recording-tab", firstThreadRef), + startBrowserRecording("recording-tab-2", secondThreadRef), + ]); + + expect(startScreencast).toHaveBeenCalledTimes(2); + expect(onFrame).toHaveBeenCalledOnce(); + expect(events).toContain("publish:recording-tab,recording-tab-2"); + expect(readActiveBrowserRecordingTabIds()).toEqual( + new Set(["recording-tab", "recording-tab-2"]), ); + expect(readActiveBrowserRecordingTabIds(firstThreadRef)).toEqual(new Set(["recording-tab"])); + expect(readActiveBrowserRecordingTabIds(secondThreadRef)).toEqual(new Set(["recording-tab-2"])); - expect(startScreencast).not.toHaveBeenCalled(); - expect(registrySet).not.toHaveBeenCalled(); + await stopBrowserRecording("recording-tab"); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set(["recording-tab-2"])); + await stopBrowserRecording("recording-tab-2"); + expect(readActiveBrowserRecordingTabIds()).toEqual(new Set()); + expect(save).toHaveBeenCalledTimes(2); }); it("does not report success for a second start while the first is still starting", async () => { let finishStartingScreencast: (() => void) | undefined; - startScreencast.mockImplementationOnce(async () => { + startScreencast.mockImplementationOnce(async (tabId: string) => { events.push("start-screencast"); + frameSubscription.listener?.({ + tabId, + data: "initial-frame", + width: 800, + height: 600, + receivedAt: "2026-06-26T00:00:00.000Z", + }); await new Promise((resolve) => { finishStartingScreencast = resolve; }); @@ -197,28 +434,27 @@ describe("browser recording", () => { expect(save).toHaveBeenCalledOnce(); }); - it("stops a screencast that finishes starting after cancellation", async () => { + it("finishes startup before stopping so an active recording yields an artifact", async () => { let finishStartingScreencast: (() => void) | undefined; startScreencast.mockImplementationOnce(async () => { events.push("start-screencast"); await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); const startPromise = startBrowserRecording("recording-tab"); - const rejectedStart = expect(startPromise).rejects.toBeInstanceOf( - BrowserRecordingOperationError, - ); await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); const stopPromise = stopBrowserRecording("recording-tab"); - await vi.waitFor(() => expect(stopScreencast).toHaveBeenCalledOnce()); + expect(stopScreencast).not.toHaveBeenCalled(); finishStartingScreencast?.(); - await rejectedStart; - await stopPromise; - expect(stopScreencast).toHaveBeenCalledTimes(2); + await startPromise; + await expect(stopPromise).resolves.toMatchObject({ tabId: "recording-tab" }); + expect(stopScreencast).toHaveBeenCalledOnce(); + expect(save).toHaveBeenCalledOnce(); expect(events.at(-1)).toBe("clear"); }); @@ -229,12 +465,10 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); const firstStart = startBrowserRecording("recording-tab"); - const rejectedFirstStart = expect(firstStart).rejects.toBeInstanceOf( - BrowserRecordingOperationError, - ); await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); const stopPromise = stopBrowserRecording("recording-tab"); @@ -243,7 +477,7 @@ describe("browser recording", () => { const startCallsBeforeFirstSettled = startScreencast.mock.calls.length; finishStartingScreencast?.(); - await rejectedFirstStart; + await firstStart; await stopPromise; await restartAfterStop; await stopBrowserRecording("recording-tab"); @@ -251,6 +485,39 @@ describe("browser recording", () => { expect(startCallsBeforeFirstSettled).toBe(1); }); + it("keeps the recording slot while a failed stop waits for startup", async () => { + let finishStartingScreencast: (() => void) | undefined; + startScreencast.mockImplementationOnce(async () => { + events.push("start-screencast"); + await new Promise((resolve) => { + finishStartingScreencast = resolve; + }); + emitRecordingFrame(); + }); + stopScreencast.mockRejectedValueOnce(new Error("initial stop failed")); + + const firstStart = startBrowserRecording("recording-tab"); + await vi.waitFor(() => expect(startScreencast).toHaveBeenCalledOnce()); + + const stopPromise = stopBrowserRecording("recording-tab"); + const rejectedStop = expect(stopPromise).rejects.toMatchObject({ + operation: "stop-screencast", + tabId: "recording-tab", + }); + expect(stopScreencast).not.toHaveBeenCalled(); + await expect(startBrowserRecording("recording-tab")).rejects.toBeInstanceOf( + BrowserRecordingConflictError, + ); + + finishStartingScreencast?.(); + await firstStart; + await rejectedStop; + expect(stopScreencast).toHaveBeenCalledOnce(); + + await startBrowserRecording("recording-tab"); + await stopBrowserRecording("recording-tab"); + }); + it("fails a stop that waits too long for startup without freeing the recording slot", async () => { vi.useFakeTimers(); let finishStartingScreencast: (() => void) | undefined; @@ -259,18 +526,16 @@ describe("browser recording", () => { await new Promise((resolve) => { finishStartingScreencast = resolve; }); + emitRecordingFrame(); }); const startPromise = startBrowserRecording("recording-tab"); - const rejectedStart = expect(startPromise).rejects.toBeInstanceOf( - BrowserRecordingOperationError, - ); expect(startScreencast).toHaveBeenCalledOnce(); const stopPromise = stopBrowserRecording("recording-tab"); await Promise.resolve(); await Promise.resolve(); - expect(stopScreencast).toHaveBeenCalledOnce(); + expect(stopScreencast).not.toHaveBeenCalled(); const rejection = expect(stopPromise).rejects.toMatchObject({ operation: "wait-startup", @@ -285,9 +550,10 @@ describe("browser recording", () => { ); finishStartingScreencast?.(); - await rejectedStart; + await startPromise; const cleanupResult = await stopBrowserRecording("recording-tab"); expect(cleanupResult).toBeNull(); + expect(stopScreencast).toHaveBeenCalledOnce(); expect(save).not.toHaveBeenCalled(); expect(events.at(-1)).toBe("clear"); }); diff --git a/apps/web/src/browser/browserRecording.ts b/apps/web/src/browser/browserRecording.ts index 01da1e4a3c1..5b88dfd4133 100644 --- a/apps/web/src/browser/browserRecording.ts +++ b/apps/web/src/browser/browserRecording.ts @@ -1,6 +1,7 @@ import type { DesktopPreviewRecordingArtifact, DesktopPreviewRecordingFrame, + ScopedThreadRef, } from "@t3tools/contracts"; import { useAtomValue } from "@effect/atom-react"; import * as Schema from "effect/Schema"; @@ -46,17 +47,6 @@ export class BrowserRecordingCanvasUnavailableError extends Schema.TaggedErrorCl } } -export class BrowserRecordingRequiresVisibleTabError extends Schema.TaggedErrorClass()( - "BrowserRecordingRequiresVisibleTabError", - { - tabId: Schema.String, - }, -) { - override get message(): string { - return `Browser recording requires tab ${this.tabId} to be visible.`; - } -} - export class BrowserRecordingOperationError extends Schema.TaggedErrorClass()( "BrowserRecordingOperationError", { @@ -66,6 +56,7 @@ export class BrowserRecordingOperationError extends Schema.TaggedErrorClass; + readonly firstFrameSize: Promise<"frame" | "cancelled">; + readonly settleFirstFrameSize: (outcome: "frame" | "cancelled") => void; + recorder: MediaRecorder | null; + mimeType: string | null; + frameSizeEstablished: boolean; + frameSequence: number; + lastDrawnFrameSequence: number; lifecycle: BrowserRecordingLifecycle; } -const activeBrowserRecordingTabIdAtom = Atom.make(null).pipe( - Atom.keepAlive, - Atom.withLabel("preview:active-browser-recording-tab"), -); +interface ActiveBrowserRecordingIndex { + readonly tabIds: ReadonlySet; +} -export function useActiveBrowserRecordingTabId(): string | null { - return useAtomValue(activeBrowserRecordingTabIdAtom); +const activeBrowserRecordingTabIdsAtom = Atom.make({ + tabIds: new Set(), +}).pipe(Atom.keepAlive, Atom.withLabel("preview:active-browser-recording-tabs")); + +export function useActiveBrowserRecordingTabIds(): ReadonlySet { + return useAtomValue(activeBrowserRecordingTabIdsAtom).tabIds; } -let active: ActiveRecording | null = null; +const activeRecordings = new Map(); let unsubscribeFrames: (() => void) | null = null; export const BROWSER_RECORDING_STARTUP_SETTLE_TIMEOUT_MS = 5_000; +export const BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS = 5_000; -export function readActiveBrowserRecordingTabId(): string | null { - return active?.tabId ?? null; +export function readActiveBrowserRecordingTabIds(threadRef?: ScopedThreadRef): ReadonlySet { + const tabIds = new Set(); + for (const recording of activeRecordings.values()) { + if ( + threadRef === undefined || + (recording.threadRef?.environmentId === threadRef.environmentId && + recording.threadRef.threadId === threadRef.threadId) + ) { + tabIds.add(recording.tabId); + } + } + return tabIds; } const preferredMimeType = (): string => { @@ -126,22 +137,52 @@ const preferredMimeType = (): string => { }; const drawFrame = (frame: DesktopPreviewRecordingFrame): void => { - const recording = active; - if (!recording || recording.tabId !== frame.tabId) return; + const recording = activeRecordings.get(frame.tabId); + if (!recording) return; + if ( + !Number.isFinite(frame.width) || + !Number.isFinite(frame.height) || + frame.width <= 0 || + frame.height <= 0 + ) { + return; + } + const width = Math.max(1, Math.round(frame.width)); + const height = Math.max(1, Math.round(frame.height)); + if (!recording.frameSizeEstablished) { + recording.canvas.width = width; + recording.canvas.height = height; + recording.frameSizeEstablished = true; + recording.settleFirstFrameSize("frame"); + } + const frameSequence = ++recording.frameSequence; const image = new Image(); image.addEventListener( "load", () => { - if (active !== recording) return; - recording.context.drawImage(image, 0, 0, recording.canvas.width, recording.canvas.height); + if ( + activeRecordings.get(frame.tabId) !== recording || + frameSequence <= recording.lastDrawnFrameSequence + ) { + return; + } + recording.lastDrawnFrameSequence = frameSequence; + const scale = Math.min(recording.canvas.width / width, recording.canvas.height / height); + const targetWidth = width * scale; + const targetHeight = height * scale; + const targetX = (recording.canvas.width - targetWidth) / 2; + const targetY = (recording.canvas.height - targetHeight) / 2; + recording.context.fillStyle = "#000000"; + recording.context.fillRect(0, 0, recording.canvas.width, recording.canvas.height); + recording.context.drawImage(image, targetX, targetY, targetWidth, targetHeight); }, { once: true }, ); image.src = `data:image/jpeg;base64,${frame.data}`; }; -const stopMediaRecorder = async (recorder: MediaRecorder): Promise => { - if (recorder.state === "inactive") return; +const stopMediaRecorder = async (recorder: MediaRecorder | null): Promise => { + if (!recorder || recorder.state === "inactive") return; const stopped = new Promise((resolve) => recorder.addEventListener("stop", () => resolve(), { once: true }), ); @@ -150,11 +191,42 @@ const stopMediaRecorder = async (recorder: MediaRecorder): Promise => { }; const clearActiveRecording = (recording: ActiveRecording): void => { - if (active !== recording) return; - active = null; - unsubscribeFrames?.(); - unsubscribeFrames = null; - appAtomRegistry.set(activeBrowserRecordingTabIdAtom, null); + if (activeRecordings.get(recording.tabId) !== recording) return; + recording.settleFirstFrameSize("cancelled"); + activeRecordings.delete(recording.tabId); + if (activeRecordings.size === 0) { + unsubscribeFrames?.(); + unsubscribeFrames = null; + } + appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { + tabIds: new Set(activeRecordings.keys()), + }); +}; + +const cleanupFailedRecordingStart = async ( + bridge: NonNullable, + recording: ActiveRecording, +): Promise => { + const errors: unknown[] = []; + try { + await bridge.recording.stopScreencast(recording.tabId); + } catch (error) { + errors.push(error); + } + try { + await stopMediaRecorder(recording.recorder); + } catch (error) { + errors.push(error); + } finally { + clearActiveRecording(recording); + } + if (errors.length === 0) return undefined; + if (errors.length === 1) return errors[0]; + return new AggregateError( + errors, + `Browser recording startup cleanup failed for tab ${recording.tabId}.`, + { cause: errors[0] }, + ); }; const recordingStartupCancelledError = ( @@ -168,7 +240,20 @@ const recordingStartupCancelledError = ( }); const isRecordingStarting = (recording: ActiveRecording): boolean => - active === recording && recording.lifecycle.phase === "starting"; + activeRecordings.get(recording.tabId) === recording && recording.lifecycle.phase === "starting"; + +const waitForFirstFrameSize = async (recording: ActiveRecording): Promise => { + if (recording.frameSizeEstablished) return true; + let timeout: ReturnType | null = null; + const outcome = await Promise.race([ + recording.firstFrameSize, + new Promise<"timeout">((resolve) => { + timeout = setTimeout(() => resolve("timeout"), BROWSER_RECORDING_FIRST_FRAME_SIZE_TIMEOUT_MS); + }), + ]); + if (timeout !== null) clearTimeout(timeout); + return outcome === "frame"; +}; const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Promise => { let timeout: ReturnType | null = null; @@ -195,20 +280,23 @@ const waitForRecordingStartupToSettle = async (recording: ActiveRecording): Prom const isStartupWaitTimeout = (error: unknown): error is BrowserRecordingOperationError => isBrowserRecordingOperationError(error) && error.operation === "wait-startup"; -export async function startBrowserRecording(tabId: string): Promise { +export async function startBrowserRecording( + tabId: string, + threadRef: ScopedThreadRef | null = null, +): Promise { const bridge = previewBridge; if (!bridge) throw new BrowserRecordingUnavailableError({ tabId }); - if (active) { - if (active.tabId === tabId && active.lifecycle.phase === "recording") { - return active.startedAt; + const activeRecording = activeRecordings.get(tabId); + if (activeRecording) { + if (activeRecording.lifecycle.phase === "recording") { + return activeRecording.startedAt; } throw new BrowserRecordingConflictError({ requestedTabId: tabId, - activeTabId: active.tabId, + activeTabId: activeRecording.tabId, }); } const surface = useBrowserSurfaceStore.getState().byTabId[tabId]; - if (!surface?.visible) throw new BrowserRecordingRequiresVisibleTabError({ tabId }); const recordingSize = surface?.content ?? surface?.rect; const canvas = document.createElement("canvas"); canvas.width = Math.max(1, recordingSize?.width ?? 1280); @@ -221,42 +309,34 @@ export async function startBrowserRecording(tabId: string): Promise { height: canvas.height, }); } - let mimeType: string; - let recorder: MediaRecorder; - try { - mimeType = preferredMimeType(); - recorder = new MediaRecorder(canvas.captureStream(12), { - mimeType, - videoBitsPerSecond: 4_000_000, - }); - } catch (cause) { - throw new BrowserRecordingOperationError({ - operation: "initialize-media-recorder", - tabId, - cause, - }); - } const startedAt = new Date().toISOString(); const chunks: Blob[] = []; let settleStartup: (() => void) | undefined; const startupSettled = new Promise((resolve) => { settleStartup = resolve; }); - recorder.addEventListener("dataavailable", (event) => { - if (event.data.size > 0) chunks.push(event.data); + let settleFirstFrameSize: ((outcome: "frame" | "cancelled") => void) | undefined; + const firstFrameSize = new Promise<"frame" | "cancelled">((resolve) => { + settleFirstFrameSize = resolve; }); const recording: ActiveRecording = { tabId, + threadRef, canvas, context, - recorder, chunks, - mimeType, startedAt, startupSettled, + firstFrameSize, + settleFirstFrameSize: (outcome) => settleFirstFrameSize?.(outcome), + recorder: null, + mimeType: null, + frameSizeEstablished: false, + frameSequence: 0, + lastDrawnFrameSequence: 0, lifecycle: { phase: "starting" }, }; - active = recording; + activeRecordings.set(tabId, recording); try { try { unsubscribeFrames ??= bridge.recording.onFrame(drawFrame); @@ -268,47 +348,23 @@ export async function startBrowserRecording(tabId: string): Promise { cause, }); } - try { - recorder.start(1_000); - } catch (cause) { - clearActiveRecording(recording); - throw new BrowserRecordingOperationError({ - operation: "start-media-recorder", - tabId, - cause, - }); - } - if (!isRecordingStarting(recording)) { - throw recordingStartupCancelledError(recording); - } try { await bridge.recording.startScreencast(tabId); } catch (cause) { if (!isRecordingStarting(recording)) { throw recordingStartupCancelledError(recording, cause); } - let cleanupCause: unknown; - try { - await stopMediaRecorder(recorder); - } catch (error) { - cleanupCause = error; - } finally { - clearActiveRecording(recording); - } + clearActiveRecording(recording); throw new BrowserRecordingOperationError({ operation: "start-screencast", tabId, - cause: - cleanupCause === undefined - ? cause - : new AggregateError( - [cause, cleanupCause], - `Browser recording start and cleanup failed for tab ${tabId}.`, - { cause }, - ), + cause, }); } - if (!isRecordingStarting(recording)) { + const throwIfStartupCancelled = async (): Promise => { + // A stop requested during startup should let startup finish so the + // caller receives a real artifact. Only replacement/removal cancels it. + if (activeRecordings.get(tabId) === recording) return; try { await bridge.recording.stopScreencast(tabId); } catch (cause) { @@ -322,9 +378,78 @@ export async function startBrowserRecording(tabId: string): Promise { ); } throw recordingStartupCancelledError(recording); + }; + await throwIfStartupCancelled(); + const hasFirstFrame = await waitForFirstFrameSize(recording); + await throwIfStartupCancelled(); + if (!hasFirstFrame) { + const cause = new Error(`No valid recording frame arrived for tab ${tabId}.`); + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + throw new BrowserRecordingOperationError({ + operation: "wait-first-frame", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser recording frame wait and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); + } + + let mimeType: string; + let recorder: MediaRecorder; + try { + mimeType = preferredMimeType(); + recorder = new MediaRecorder(canvas.captureStream(12), { + mimeType, + videoBitsPerSecond: 4_000_000, + }); + recording.mimeType = mimeType; + recording.recorder = recorder; + recorder.addEventListener("dataavailable", (event) => { + if (event.data.size > 0) chunks.push(event.data); + }); + } catch (cause) { + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + throw new BrowserRecordingOperationError({ + operation: "initialize-media-recorder", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser recording initialization and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); } - recording.lifecycle = { phase: "recording" }; - appAtomRegistry.set(activeBrowserRecordingTabIdAtom, tabId); + try { + recorder.start(1_000); + } catch (cause) { + const cleanupCause = await cleanupFailedRecordingStart(bridge, recording); + throw new BrowserRecordingOperationError({ + operation: "start-media-recorder", + tabId, + cause: + cleanupCause === undefined + ? cause + : new AggregateError( + [cause, cleanupCause], + `Browser media recorder start and cleanup failed for tab ${tabId}.`, + { cause }, + ), + }); + } + if (recording.lifecycle.phase === "starting") { + recording.lifecycle = { phase: "recording" }; + } + appAtomRegistry.set(activeBrowserRecordingTabIdsAtom, { + tabIds: new Set(activeRecordings.keys()), + }); return startedAt; } finally { settleStartup?.(); @@ -334,12 +459,16 @@ export async function startBrowserRecording(tabId: string): Promise { const finalizeBrowserRecording = async ( bridge: NonNullable, recording: ActiveRecording, -): Promise => { +): Promise => { const { tabId } = recording; let result: - | { readonly _tag: "Success"; readonly artifact: DesktopPreviewRecordingArtifact } + | { + readonly _tag: "Success"; + readonly artifact: DesktopPreviewRecordingArtifact | null; + } | { readonly _tag: "Failure"; readonly error: unknown }; try { + await waitForRecordingStartupToSettle(recording); try { await bridge.recording.stopScreencast(tabId); } catch (cause) { @@ -349,30 +478,33 @@ const finalizeBrowserRecording = async ( cause, }); } - await waitForRecordingStartupToSettle(recording); - try { - await stopMediaRecorder(recording.recorder); - } catch (cause) { - throw new BrowserRecordingOperationError({ - operation: "stop-media-recorder", - tabId, - cause, - }); - } - try { - const blob = new Blob(recording.chunks, { type: recording.mimeType }); - const artifact = await bridge.recording.save( - tabId, - recording.mimeType, - new Uint8Array(await blob.arrayBuffer()), - ); - result = { _tag: "Success", artifact }; - } catch (cause) { - throw new BrowserRecordingOperationError({ - operation: "save-artifact", - tabId, - cause, - }); + if (!recording.recorder || !recording.mimeType) { + result = { _tag: "Success", artifact: null }; + } else { + try { + await stopMediaRecorder(recording.recorder); + } catch (cause) { + throw new BrowserRecordingOperationError({ + operation: "stop-media-recorder", + tabId, + cause, + }); + } + try { + const blob = new Blob(recording.chunks, { type: recording.mimeType }); + const artifact = await bridge.recording.save( + tabId, + recording.mimeType, + new Uint8Array(await blob.arrayBuffer()), + ); + result = { _tag: "Success", artifact }; + } catch (cause) { + throw new BrowserRecordingOperationError({ + operation: "save-artifact", + tabId, + cause, + }); + } } } catch (error) { result = { _tag: "Failure", error }; @@ -434,14 +566,14 @@ export function stopBrowserRecording( tabId: string, ): Promise { const bridge = previewBridge; - const recording = active; - if (!bridge || !recording || recording.tabId !== tabId) return Promise.resolve(null); + const recording = activeRecordings.get(tabId); + if (!bridge || !recording) return Promise.resolve(null); if (recording.lifecycle.phase === "stopping") return recording.lifecycle.stopPromise; const stopPromise = Promise.resolve() .then(() => finalizeBrowserRecording(bridge, recording)) .catch((error) => { - if (isStartupWaitTimeout(error) && active === recording) { + if (isStartupWaitTimeout(error) && activeRecordings.get(recording.tabId) === recording) { const cleanupAfterStartup = recording.startupSettled.then(() => discardBrowserRecording(bridge, recording), ); diff --git a/apps/web/src/browser/browserRecordingScope.test.ts b/apps/web/src/browser/browserRecordingScope.test.ts index 6b3adf2219e..304adf4f19e 100644 --- a/apps/web/src/browser/browserRecordingScope.test.ts +++ b/apps/web/src/browser/browserRecordingScope.test.ts @@ -3,14 +3,41 @@ import { describe, expect, it } from "vite-plus/test"; import { resolveBrowserRecordingStopTarget } from "./browserRecordingScope"; describe("resolveBrowserRecordingStopTarget", () => { - it("stops the active recording when no explicit tab was requested", () => { - expect(resolveBrowserRecordingStopTarget("tab-a")).toBe("tab-a"); - expect(resolveBrowserRecordingStopTarget("tab-b")).toBe("tab-b"); - expect(resolveBrowserRecordingStopTarget(null)).toBeNull(); + it("stops the only active recording when the implicit browser target changed", () => { + expect(resolveBrowserRecordingStopTarget(new Set(["tab-recording"]), "tab-browsing")).toBe( + "tab-recording", + ); }); - it("only stops an explicitly requested tab when it owns the recording", () => { - expect(resolveBrowserRecordingStopTarget("tab-a", "tab-a")).toBe("tab-a"); - expect(resolveBrowserRecordingStopTarget("tab-a", "tab-b")).toBeNull(); + it("prefers an implicit target that is actively recording", () => { + expect( + resolveBrowserRecordingStopTarget( + new Set(["tab-recording-a", "tab-recording-b"]), + "tab-recording-b", + ), + ).toBe("tab-recording-b"); + }); + + it("does not guess when multiple recordings are active and the implicit target is not one", () => { + expect( + resolveBrowserRecordingStopTarget( + new Set(["tab-recording-a", "tab-recording-b"]), + "tab-browsing", + ), + ).toBeNull(); + }); + + it("only stops an explicitly requested tab when that tab is recording", () => { + const activeTabIds = new Set(["tab-recording"]); + expect(resolveBrowserRecordingStopTarget(activeTabIds, "tab-browsing", "tab-recording")).toBe( + "tab-recording", + ); + expect(resolveBrowserRecordingStopTarget(activeTabIds, "tab-recording", "tab-browsing")).toBe( + null, + ); + }); + + it("returns null when no matching recording is active", () => { + expect(resolveBrowserRecordingStopTarget(new Set(), "tab-browsing")).toBeNull(); }); }); diff --git a/apps/web/src/browser/browserRecordingScope.ts b/apps/web/src/browser/browserRecordingScope.ts index 92f58016aa1..ffbcc19a545 100644 --- a/apps/web/src/browser/browserRecordingScope.ts +++ b/apps/web/src/browser/browserRecordingScope.ts @@ -1,7 +1,14 @@ export function resolveBrowserRecordingStopTarget( - activeTabId: string | null, - requestedTabId?: string, + activeTabIds: ReadonlySet, + implicitTabId: string | null, + explicitTabId?: string, ): string | null { - if (activeTabId === null) return null; - return requestedTabId === undefined || requestedTabId === activeTabId ? activeTabId : null; + if (explicitTabId !== undefined) { + return activeTabIds.has(explicitTabId) ? explicitTabId : null; + } + if (implicitTabId !== null && activeTabIds.has(implicitTabId)) { + return implicitTabId; + } + if (activeTabIds.size !== 1) return null; + return activeTabIds.values().next().value ?? null; } diff --git a/apps/web/src/browser/browserSurfaceStore.test.ts b/apps/web/src/browser/browserSurfaceStore.test.ts index ecfce8cb432..0379f1382e7 100644 --- a/apps/web/src/browser/browserSurfaceStore.test.ts +++ b/apps/web/src/browser/browserSurfaceStore.test.ts @@ -11,6 +11,62 @@ describe("browserSurfaceStore", () => { useBrowserSurfaceStore.setState({ byTabId: {} }); }); + it("freezes the source content dimensions for a fitted presentation", () => { + const tabId = "fitted-browser-surface"; + const sourceOwner = Symbol("source"); + const sourceContent = { + x: 10, + y: 20, + width: 1_280, + height: 720, + scale: 1, + scrollLeft: 0, + scrollTop: 0, + }; + useBrowserSurfaceStore.getState().claim(tabId, sourceOwner, false); + useBrowserSurfaceStore.getState().presentContent(tabId, sourceContent); + + const fittedLease = acquireBrowserSurface(tabId, true); + useBrowserSurfaceStore.getState().presentContent(tabId, { + ...sourceContent, + width: 360, + height: 203, + scale: 0.28125, + }); + + expect(useBrowserSurfaceStore.getState().byTabId[tabId]?.fittedSourceContent).toEqual( + sourceContent, + ); + fittedLease.release(); + }); + + it("freezes the first content dimensions when fitting starts before the browser is measured", () => { + const tabId = "pending-fitted-browser-surface"; + const fittedLease = acquireBrowserSurface(tabId, true); + const sourceContent = { + x: 0, + y: 0, + width: 1_280, + height: 720, + scale: 1, + scrollLeft: 0, + scrollTop: 0, + }; + + useBrowserSurfaceStore.getState().presentContent(tabId, sourceContent); + useBrowserSurfaceStore.getState().presentContent(tabId, { + ...sourceContent, + width: 320, + height: 180, + scale: 0.25, + }); + + expect(useBrowserSurfaceStore.getState().byTabId[tabId]?.fittedSourceContent).toEqual( + sourceContent, + ); + fittedLease.release(); + }); + it("tracks content dimensions for a browser that has never been visible", () => { const tabId = "hidden-browser-surface-content-test"; useBrowserSurfaceStore.getState().presentContent(tabId, { @@ -36,8 +92,26 @@ describe("browserSurfaceStore", () => { expect( resolveBrowserSurfacePanelRect( { - hidden: { rect: staleRect, visible: false, content: null, updatedAt: 1, owner: null }, - active: { rect: liveRect, visible: true, content: null, updatedAt: 2, owner: null }, + hidden: { + rect: staleRect, + visible: false, + content: null, + fittedSourceContent: null, + fitSourceContent: false, + cornerRadius: 0, + updatedAt: 1, + owner: null, + }, + active: { + rect: liveRect, + visible: true, + content: null, + fittedSourceContent: null, + fitSourceContent: false, + cornerRadius: 0, + updatedAt: 2, + owner: null, + }, }, "hidden", ), @@ -53,7 +127,7 @@ describe("browserSurfaceStore", () => { const liveLease = acquireBrowserSurface(tabId); liveLease.present(liveRect, true); - staleLease.present(staleRect, true); + expect(staleLease.present(staleRect, true)).toBe(false); staleLease.release(); expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({ @@ -75,4 +149,27 @@ describe("browserSurfaceStore", () => { owner: null, }); }); + + it("clears fitted presentation state when its lease is released", () => { + const tabId = "released-fitted-browser-surface"; + const fittedLease = acquireBrowserSurface(tabId, true); + useBrowserSurfaceStore.getState().presentContent(tabId, { + x: 0, + y: 0, + width: 1_280, + height: 800, + scale: 1, + scrollLeft: 0, + scrollTop: 0, + }); + + fittedLease.release(); + + expect(useBrowserSurfaceStore.getState().byTabId[tabId]).toMatchObject({ + fittedSourceContent: null, + fitSourceContent: false, + owner: null, + visible: false, + }); + }); }); diff --git a/apps/web/src/browser/browserSurfaceStore.ts b/apps/web/src/browser/browserSurfaceStore.ts index 58012a11a30..c9b7cae544c 100644 --- a/apps/web/src/browser/browserSurfaceStore.ts +++ b/apps/web/src/browser/browserSurfaceStore.ts @@ -11,6 +11,9 @@ export interface BrowserSurfacePresentation { readonly rect: BrowserSurfaceRect | null; readonly visible: boolean; readonly content: BrowserSurfaceContentPresentation | null; + readonly fittedSourceContent: BrowserSurfaceContentPresentation | null; + readonly fitSourceContent: boolean; + readonly cornerRadius: number; readonly updatedAt: number; readonly owner: symbol | null; } @@ -27,19 +30,20 @@ export interface BrowserSurfaceContentPresentation { interface BrowserSurfaceStoreState { readonly byTabId: Record; - readonly claim: (tabId: string, owner: symbol) => void; + readonly claim: (tabId: string, owner: symbol, fitSourceContent: boolean) => void; readonly present: ( tabId: string, owner: symbol, rect: BrowserSurfaceRect, visible: boolean, + cornerRadius: number, ) => void; readonly presentContent: (tabId: string, content: BrowserSurfaceContentPresentation) => void; readonly release: (tabId: string, owner: symbol) => void; } export interface BrowserSurfaceLease { - readonly present: (rect: BrowserSurfaceRect, visible: boolean) => void; + readonly present: (rect: BrowserSurfaceRect, visible: boolean, cornerRadius?: number) => boolean; readonly release: () => void; } @@ -72,7 +76,7 @@ const rectEquals = (left: BrowserSurfaceRect | null, right: BrowserSurfaceRect): export const useBrowserSurfaceStore = create()((set) => ({ byTabId: {}, - claim: (tabId, owner) => + claim: (tabId, owner, fitSourceContent) => set((state) => { const current = state.byTabId[tabId]; if (current?.owner === owner) return state; @@ -83,21 +87,31 @@ export const useBrowserSurfaceStore = create()((set) = rect: current?.rect ?? null, visible: false, content: current?.content ?? null, + fittedSourceContent: fitSourceContent ? (current?.content ?? null) : null, + fitSourceContent, + cornerRadius: current?.cornerRadius ?? 0, updatedAt: Date.now(), owner, }, }, }; }), - present: (tabId, owner, rect, visible) => + present: (tabId, owner, rect, visible, cornerRadius) => set((state) => { const current = state.byTabId[tabId]; if (current?.owner !== owner) return state; - if (current && current.visible === visible && rectEquals(current.rect, rect)) return state; + if ( + current && + current.visible === visible && + current.cornerRadius === cornerRadius && + rectEquals(current.rect, rect) + ) { + return state; + } return { byTabId: { ...state.byTabId, - [tabId]: { ...current, rect, visible, updatedAt: Date.now() }, + [tabId]: { ...current, rect, visible, cornerRadius, updatedAt: Date.now() }, }, }; }), @@ -112,6 +126,9 @@ export const useBrowserSurfaceStore = create()((set) = rect: null, visible: false, content, + fittedSourceContent: null, + fitSourceContent: false, + cornerRadius: 0, updatedAt: Date.now(), owner: null, }, @@ -134,7 +151,15 @@ export const useBrowserSurfaceStore = create()((set) = return { byTabId: { ...state.byTabId, - [tabId]: { ...current, content, updatedAt: Date.now() }, + [tabId]: { + ...current, + content, + fittedSourceContent: + current.fitSourceContent && current.fittedSourceContent === null + ? content + : current.fittedSourceContent, + updatedAt: Date.now(), + }, }, }; }), @@ -145,21 +170,33 @@ export const useBrowserSurfaceStore = create()((set) = return { byTabId: { ...state.byTabId, - [tabId]: { ...current, visible: false, updatedAt: Date.now(), owner: null }, + [tabId]: { + ...current, + visible: false, + fittedSourceContent: null, + fitSourceContent: false, + updatedAt: Date.now(), + owner: null, + }, }, }; }), })); -export function acquireBrowserSurface(tabId: string): BrowserSurfaceLease { +export function acquireBrowserSurface( + tabId: string, + fitSourceContent = false, +): BrowserSurfaceLease { const owner = Symbol(`browser-surface:${tabId}`); let released = false; - useBrowserSurfaceStore.getState().claim(tabId, owner); + useBrowserSurfaceStore.getState().claim(tabId, owner, fitSourceContent); return { - present: (rect, visible) => { - if (released) return; - useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible); + present: (rect, visible, cornerRadius = 0) => { + if (released) return false; + if (useBrowserSurfaceStore.getState().byTabId[tabId]?.owner !== owner) return false; + useBrowserSurfaceStore.getState().present(tabId, owner, rect, visible, cornerRadius); + return true; }, release: () => { if (released) return; diff --git a/apps/web/src/browser/desktopTabLifetime.test.ts b/apps/web/src/browser/desktopTabLifetime.test.ts index 1e3b1632bcc..8f7b2ca7d65 100644 --- a/apps/web/src/browser/desktopTabLifetime.test.ts +++ b/apps/web/src/browser/desktopTabLifetime.test.ts @@ -1,20 +1,32 @@ -import { beforeEach, describe, expect, it, vi } from "vite-plus/test"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; -const { closeTab, createTab } = vi.hoisted(() => ({ - closeTab: vi.fn(async () => undefined), +const { closeTab, createTab, stopBrowserRecording } = vi.hoisted(() => ({ + closeTab: vi.fn<(tabId: string) => Promise>(async () => undefined), createTab: vi.fn<() => Promise>(), + stopBrowserRecording: vi.fn(async () => null), })); vi.mock("~/components/preview/previewBridge", () => ({ previewBridge: { closeTab, createTab }, })); +vi.mock("./browserRecording", () => ({ + stopBrowserRecording, +})); + import { acquireDesktopTab } from "./desktopTabLifetime"; describe("desktopTabLifetime", () => { beforeEach(() => { closeTab.mockClear(); createTab.mockClear(); + stopBrowserRecording.mockClear(); + vi.stubGlobal("window", globalThis); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); }); it("shares tab creation readiness across concurrent leases", async () => { @@ -42,4 +54,53 @@ describe("desktopTabLifetime", () => { await first.ready; expect(ready).toBe(true); }); + + it("stops recording before closing the final desktop tab lease", async () => { + vi.useFakeTimers(); + let resolveStop: (() => void) | undefined; + stopBrowserRecording.mockReturnValueOnce( + new Promise((resolve) => { + resolveStop = () => resolve(null); + }), + ); + createTab.mockResolvedValueOnce(undefined); + + const lease = acquireDesktopTab("tab_recording_cleanup"); + await lease.ready; + lease.release(); + await vi.advanceTimersByTimeAsync(0); + + expect(stopBrowserRecording).toHaveBeenCalledWith("tab_recording_cleanup"); + expect(closeTab).not.toHaveBeenCalled(); + + resolveStop?.(); + await Promise.resolve(); + await Promise.resolve(); + expect(closeTab).toHaveBeenCalledWith("tab_recording_cleanup"); + }); + + it("waits for an in-flight close before recreating a reacquired tab", async () => { + vi.useFakeTimers(); + let resolveClose: (() => void) | undefined; + createTab.mockResolvedValue(undefined); + closeTab.mockReturnValueOnce( + new Promise((resolve) => { + resolveClose = resolve; + }), + ); + + const initial = acquireDesktopTab("tab_close_reacquire"); + await initial.ready; + initial.release(); + await vi.advanceTimersByTimeAsync(0); + + expect(closeTab).toHaveBeenCalledWith("tab_close_reacquire"); + + const reacquired = acquireDesktopTab("tab_close_reacquire"); + expect(createTab).toHaveBeenCalledTimes(1); + + resolveClose?.(); + await reacquired.ready; + expect(createTab).toHaveBeenCalledTimes(2); + }); }); diff --git a/apps/web/src/browser/desktopTabLifetime.ts b/apps/web/src/browser/desktopTabLifetime.ts index d621f6dc30c..98dffda0ea0 100644 --- a/apps/web/src/browser/desktopTabLifetime.ts +++ b/apps/web/src/browser/desktopTabLifetime.ts @@ -1,5 +1,7 @@ import { previewBridge } from "~/components/preview/previewBridge"; +import { stopBrowserRecording } from "./browserRecording"; + interface DesktopTabLease { references: number; closeTimer: number | null; @@ -7,6 +9,26 @@ interface DesktopTabLease { } const leases = new Map(); +const pendingTabOperations = new Map>(); + +const enqueueDesktopTabOperation = ( + tabId: string, + operation: () => Promise | void, +): Promise => { + const previous = pendingTabOperations.get(tabId); + const pending = previous + ? previous.catch(() => undefined).then(operation) + : Promise.resolve(operation()); + pendingTabOperations.set(tabId, pending); + void pending + .finally(() => { + if (pendingTabOperations.get(tabId) === pending) { + pendingTabOperations.delete(tabId); + } + }) + .catch(() => undefined); + return pending; +}; export interface AcquiredDesktopTab { readonly ready: Promise; @@ -19,7 +41,7 @@ export function acquireDesktopTab(tabId: string): AcquiredDesktopTab { ({ references: 0, closeTimer: null, - ready: previewBridge?.createTab(tabId) ?? Promise.resolve(), + ready: enqueueDesktopTabOperation(tabId, () => previewBridge?.createTab(tabId)), } satisfies DesktopTabLease); if (current.closeTimer !== null) window.clearTimeout(current.closeTimer); current.references += 1; @@ -37,7 +59,10 @@ export function acquireDesktopTab(tabId: string): AcquiredDesktopTab { const latest = leases.get(tabId); if (!latest || latest.references > 0) return; leases.delete(tabId); - void previewBridge?.closeTab(tabId); + void enqueueDesktopTabOperation(tabId, async () => { + await stopBrowserRecording(tabId).catch(() => null); + await previewBridge?.closeTab(tabId); + }).catch(() => undefined); }, 0); }, }; diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts index 826684bb06f..d0298dcdee7 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.test.ts @@ -23,6 +23,23 @@ describe("resolveHostedBrowserWebviewWrapperStyle", () => { }); }); + it("clips a floating webview to the mini-player frame", () => { + expect( + resolveHostedBrowserWebviewWrapperStyle({ + active: true, + cornerRadius: 12, + rect: { x: 12, y: 34, width: 360, height: 203 }, + hiddenSize: { width: 1280, height: 800 }, + }), + ).toMatchObject({ + left: 12, + top: 34, + width: 360, + height: 203, + borderRadius: 12, + }); + }); + it("keeps an inactive webview paintable while moving it offscreen", () => { const style = resolveHostedBrowserWebviewWrapperStyle({ active: false, diff --git a/apps/web/src/browser/hostedBrowserWebviewStyle.ts b/apps/web/src/browser/hostedBrowserWebviewStyle.ts index 4dade986e1f..f96f4af0462 100644 --- a/apps/web/src/browser/hostedBrowserWebviewStyle.ts +++ b/apps/web/src/browser/hostedBrowserWebviewStyle.ts @@ -12,6 +12,7 @@ export interface HostedBrowserWebviewWrapperStyle { readonly height: number; readonly zIndex: number; readonly pointerEvents: "auto" | "none"; + readonly borderRadius?: number; readonly visibility?: "visible"; } @@ -19,10 +20,11 @@ export const HIDDEN_BROWSER_WEBVIEW_OFFSET = -100_000; export function resolveHostedBrowserWebviewWrapperStyle(input: { readonly active: boolean; + readonly cornerRadius?: number; readonly rect: BrowserSurfaceRect | null; readonly hiddenSize: HostedBrowserWebviewSize; }): HostedBrowserWebviewWrapperStyle { - const { active, hiddenSize, rect } = input; + const { active, cornerRadius = 0, hiddenSize, rect } = input; if (active && rect) { return { left: rect.x, @@ -31,6 +33,7 @@ export function resolveHostedBrowserWebviewWrapperStyle(input: { height: rect.height, zIndex: 30, pointerEvents: "auto", + ...(cornerRadius > 0 ? { borderRadius: cornerRadius } : {}), }; } diff --git a/apps/web/src/browser/webviewCrashRecovery.test.ts b/apps/web/src/browser/webviewCrashRecovery.test.ts new file mode 100644 index 00000000000..a13e6ff06f6 --- /dev/null +++ b/apps/web/src/browser/webviewCrashRecovery.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, + planWebviewCrashRecovery, + WEBVIEW_CRASH_RECOVERY_WINDOW_MS, +} from "./webviewCrashRecovery"; + +describe("planWebviewCrashRecovery", () => { + it("backs off and stops after a bounded number of rapid crashes", () => { + const first = planWebviewCrashRecovery(INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, 1_000); + expect(first).not.toBeNull(); + expect(first?.delayMs).toBe(250); + + const second = planWebviewCrashRecovery(first!.state, 1_100); + expect(second).not.toBeNull(); + expect(second?.delayMs).toBe(500); + + const third = planWebviewCrashRecovery(second!.state, 1_200); + expect(third).not.toBeNull(); + expect(third?.delayMs).toBe(1_000); + + expect(planWebviewCrashRecovery(third!.state, 1_300)).toBeNull(); + }); + + it("allows recovery again after the crash window expires", () => { + const first = planWebviewCrashRecovery(INITIAL_WEBVIEW_CRASH_RECOVERY_STATE, 1_000)!; + const second = planWebviewCrashRecovery(first.state, 1_100)!; + const third = planWebviewCrashRecovery(second.state, 1_200)!; + + expect(planWebviewCrashRecovery(third.state, 1_000 + WEBVIEW_CRASH_RECOVERY_WINDOW_MS)).toEqual( + { + delayMs: 250, + state: { + attempts: 1, + windowStartedAt: 1_000 + WEBVIEW_CRASH_RECOVERY_WINDOW_MS, + }, + }, + ); + }); +}); diff --git a/apps/web/src/browser/webviewCrashRecovery.ts b/apps/web/src/browser/webviewCrashRecovery.ts new file mode 100644 index 00000000000..2267f4a812d --- /dev/null +++ b/apps/web/src/browser/webviewCrashRecovery.ts @@ -0,0 +1,38 @@ +export const WEBVIEW_CRASH_RECOVERY_WINDOW_MS = 30_000; +export const WEBVIEW_CRASH_RECOVERY_MAX_ATTEMPTS = 3; +export const WEBVIEW_CRASH_RECOVERY_BASE_DELAY_MS = 250; + +export interface WebviewCrashRecoveryState { + readonly attempts: number; + readonly windowStartedAt: number | null; +} + +export interface WebviewCrashRecoveryPlan { + readonly delayMs: number; + readonly state: WebviewCrashRecoveryState; +} + +export const INITIAL_WEBVIEW_CRASH_RECOVERY_STATE: WebviewCrashRecoveryState = { + attempts: 0, + windowStartedAt: null, +}; + +export function planWebviewCrashRecovery( + state: WebviewCrashRecoveryState, + now: number, +): WebviewCrashRecoveryPlan | null { + const startsNewWindow = + state.windowStartedAt === null || + now - state.windowStartedAt >= WEBVIEW_CRASH_RECOVERY_WINDOW_MS; + const attempts = startsNewWindow ? 0 : state.attempts; + if (attempts >= WEBVIEW_CRASH_RECOVERY_MAX_ATTEMPTS) return null; + + const nextAttempts = attempts + 1; + return { + delayMs: WEBVIEW_CRASH_RECOVERY_BASE_DELAY_MS * 2 ** attempts, + state: { + attempts: nextAttempts, + windowStartedAt: startsNewWindow ? now : state.windowStartedAt, + }, + }; +} diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index ab1256cddb3..918e2c92bb9 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -131,8 +131,13 @@ import { } from "../previewStateStore"; import { addBrowserSurface } from "./preview/addBrowserSurface"; import { closePreviewSession } from "./preview/closePreviewSession"; +import { ThreadPreviewMiniPlayer } from "./preview/ThreadPreviewMiniPlayer"; import { subscribePreviewAction } from "./preview/previewActionBus"; import { getConfiguredPreviewUrls } from "./preview/previewEmptyStateLogic"; +import { + selectThreadPreviewMiniPlayer, + usePreviewMiniPlayerStore, +} from "../previewMiniPlayerStore"; import { RightPanelTabs } from "./RightPanelTabs"; import { DiffWorkerPoolProvider } from "./DiffWorkerPoolProvider"; import { BranchToolbar } from "./BranchToolbar"; @@ -1484,6 +1489,9 @@ function ChatViewContent(props: ChatViewProps) { const activeFileSurface = activeRightPanelSurface?.kind === "file" ? activeRightPanelSurface : null; const activePreviewState = useThreadPreviewState(activeThreadRef); + const activePreviewMiniPlayer = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayer(state.byThreadKey, activeThreadRef), + ); const panelTerminalIds = useMemo( () => new Set( @@ -1507,6 +1515,24 @@ function ChatViewContent(props: ChatViewProps) { .reconcileBrowserSurfaces(activeThreadRef, Object.keys(activePreviewState.sessions)); }, [activePreviewState.sessions, activeThreadRef]); + useEffect(() => { + if (!activeThreadRef || !activePreviewMiniPlayer) return; + const miniTabStillExists = Boolean(activePreviewState.sessions[activePreviewMiniPlayer.tabId]); + const sameTabOpenInPanel = + previewPanelOpen && + activeRightPanelSurface?.kind === "preview" && + activeRightPanelSurface.resourceId === activePreviewMiniPlayer.tabId; + if (!miniTabStillExists || sameTabOpenInPanel) { + usePreviewMiniPlayerStore.getState().close(activeThreadRef); + } + }, [ + activePreviewMiniPlayer, + activePreviewState.sessions, + activeRightPanelSurface, + activeThreadRef, + previewPanelOpen, + ]); + const planSidebarOpen = activeRightPanelKind === "plan"; const existingOpenTerminalThreadKeys = useMemo(() => { @@ -5907,6 +5933,15 @@ function ChatViewContent(props: ChatViewProps) {
+ {activeThreadRef && activePreviewMiniPlayer ? ( + + ) : null} + diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 27383b29f19..165fb6136aa 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -29,10 +29,10 @@ import { reconcilePreviewServerSessions, updatePreviewServerSnapshot, } from "~/previewStateStore"; -import { useRightPanelStore } from "~/rightPanelStore"; +import { usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; import { resolveBrowserNavigationTarget } from "~/browser/browserTargetResolver"; import { - readActiveBrowserRecordingTabId, + readActiveBrowserRecordingTabIds, startBrowserRecording, stopBrowserRecording, } from "~/browser/browserRecording"; @@ -53,7 +53,10 @@ import { PreviewAutomationTargetUnavailableError, PreviewAutomationViewportTimeoutError, } from "./previewAutomationErrors"; -import { previewAutomationOpenNeedsOverlay } from "./previewAutomationOpenReadiness"; +import { + previewAutomationOpenNeedsOverlay, + shouldOpenPreviewMiniPlayer, +} from "./previewAutomationOpenReadiness"; import { createPreviewAutomationRequestConsumerAtom } from "./previewAutomationRequestConsumer"; import { createPreviewAutomationClientId } from "./previewAutomationClientId"; import { @@ -319,7 +322,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) if (result._tag === "Failure") { return raiseAtomCommandFailure(result); } - reconcilePreviewServerSessions(threadRef, result.value.sessions); + reconcilePreviewServerSessions(threadRef, result.value); state = readThreadPreviewState(threadRef); } tabId = request.tabId ?? state.snapshot?.tabId ?? null; @@ -378,8 +381,8 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) activeSnapshot = snapshot; tabId = activeTabId; } - if (input.show ?? true) { - useRightPanelStore.getState().openBrowser(threadRef, activeTabId); + if (shouldOpenPreviewMiniPlayer(input)) { + usePreviewMiniPlayerStore.getState().open(threadRef, activeTabId); } if (activeSnapshot && previewAutomationOpenNeedsOverlay(input, activeSnapshot)) { await waitForDesktopOverlay( @@ -510,7 +513,7 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "recordingStart": { const ready = await requireReadyTab(); - const startedAt = await startBrowserRecording(ready.tabId); + const startedAt = await startBrowserRecording(ready.tabId, threadRef); return { tabId: ready.tabId, recording: true, @@ -518,11 +521,13 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) }; } case "recordingStop": { - const recordingTabId = readActiveBrowserRecordingTabId(); + const activeTabIds = readActiveBrowserRecordingTabIds(threadRef); const stopTabId = resolveBrowserRecordingStopTarget( - recordingTabId, + activeTabIds, + tabId, request.tabIdExplicit ? request.tabId : undefined, ); + tabId = stopTabId ?? tabId; const artifact = stopTabId ? await stopBrowserRecording(stopTabId) : null; if (!artifact) { return raisePreviewAutomationHostError( diff --git a/apps/web/src/components/preview/PreviewChromeRow.tsx b/apps/web/src/components/preview/PreviewChromeRow.tsx index f2c3fdfea76..8044b472264 100644 --- a/apps/web/src/components/preview/PreviewChromeRow.tsx +++ b/apps/web/src/components/preview/PreviewChromeRow.tsx @@ -4,6 +4,7 @@ import { Camera, ExternalLink, MousePointerClick, + PictureInPicture2, RotateCw, } from "lucide-react"; import { @@ -40,6 +41,9 @@ interface Props { onCapture?: ((record: boolean) => void) | undefined; captureDisabled?: boolean | undefined; recording?: boolean | undefined; + onPictureInPicture?: (() => void) | undefined; + pictureInPicture?: boolean | undefined; + pictureInPictureDisabled?: boolean | undefined; /** * When provided, renders an annotation-mode toggle button to the right of * the URL input. Pressed while annotation mode is active (button shows in `pressed` @@ -77,6 +81,9 @@ export function PreviewChromeRow({ onCapture, captureDisabled, recording, + onPictureInPicture, + pictureInPicture, + pictureInPictureDisabled, onPickElement, pickActive, pickDisabled, @@ -274,6 +281,30 @@ export function PreviewChromeRow({ ) : null} + {onPictureInPicture ? ( + + + } + > + + + + {pictureInPicture ? "Close floating preview" : "Float preview over chat"} + + + ) : null} {trailingActions} {loadProgress > 0 ? ( diff --git a/apps/web/src/components/preview/PreviewMoreMenu.tsx b/apps/web/src/components/preview/PreviewMoreMenu.tsx index 28fc2e22232..a98d33304e8 100644 --- a/apps/web/src/components/preview/PreviewMoreMenu.tsx +++ b/apps/web/src/components/preview/PreviewMoreMenu.tsx @@ -46,6 +46,10 @@ interface Props { deviceToolbarVisible: boolean; /** Switches between fill-panel mode and a fixed responsive viewport. */ onToggleDeviceToolbar: () => void; + /** Whether the separate native always-on-top preview window is open. */ + nativePictureInPicture: boolean; + /** Toggles the optional native always-on-top preview window. */ + onNativePictureInPicture: () => void; } /** @@ -60,6 +64,8 @@ export function PreviewMoreMenu({ colorScheme, deviceToolbarVisible, onToggleDeviceToolbar, + nativePictureInPicture, + onNativePictureInPicture, }: Props) { if (!previewBridge) return null; const bridge = previewBridge; @@ -93,6 +99,11 @@ export function PreviewMoreMenu({ Open DevTools + + {nativePictureInPicture + ? "Close separate preview window" + : "Open separate preview window"} + {deviceToolbarVisible ? "Hide device toolbar" : "Show device toolbar"} diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 61111025fa4..2532b8c89ed 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -8,6 +8,16 @@ const mocks = vi.hoisted(() => ({ readPreparedConnection: vi.fn(() => ({ httpBaseUrl: "http://172.25.85.75:3773" })), submittedUrl: null as ((url: string) => void) | null, emptyStateUrl: null as ((url: string) => void) | null, + togglePictureInPicture: null as (() => void) | null, + toggleNativePictureInPicture: null as (() => void) | null, + pictureInPicturePressed: false, + miniPlayerTabId: null as string | null, + openMiniPlayer: vi.fn(), + closeMiniPlayer: vi.fn(), + closeRightPanel: vi.fn(), + openPictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), + closePictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), + pictureInPicture: false, showEmptyState: false, })); @@ -36,10 +46,12 @@ vi.mock("~/previewStateStore", () => ({ activeTabId: "tab-1", desktopByTabId: { "tab-1": { + hasWebContents: true, canGoBack: false, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: mocks.pictureInPicture, colorScheme: "system", controller: "none", }, @@ -80,7 +92,7 @@ vi.mock("~/state/use-atom-command", () => ({ vi.mock("~/browser/browserRecording", () => ({ startBrowserRecording: vi.fn(), stopBrowserRecording: vi.fn(), - useActiveBrowserRecordingTabId: () => null, + useActiveBrowserRecordingTabIds: () => new Set(), })); vi.mock("~/browser/browserSurfaceStore", () => ({ @@ -89,18 +101,69 @@ vi.mock("~/browser/browserSurfaceStore", () => ({ ) => select({ byTabId: {} }), })); +vi.mock("~/previewMiniPlayerStore", () => { + const usePreviewMiniPlayerStore = Object.assign( + (select: (state: unknown) => unknown) => + select({ + byThreadKey: mocks.miniPlayerTabId + ? { + "environment-1:thread-1": { + tabId: mocks.miniPlayerTabId, + position: null, + }, + } + : {}, + }), + { + getState: () => ({ + open: mocks.openMiniPlayer, + close: mocks.closeMiniPlayer, + }), + }, + ); + return { + selectThreadPreviewMiniPlayer: ( + byThreadKey: Record, + ) => byThreadKey["environment-1:thread-1"] ?? null, + usePreviewMiniPlayerStore, + }; +}); + +vi.mock("~/rightPanelStore", () => ({ + useRightPanelStore: { + getState: () => ({ close: mocks.closeRightPanel }), + }, +})); + vi.mock("~/components/ui/toast", () => ({ stackedThreadToast: vi.fn(), toastManager: { add: vi.fn() }, })); vi.mock("./previewBridge", () => ({ - previewBridge: { navigate: mocks.navigate }, + previewBridge: { + navigate: mocks.navigate, + pictureInPicture: { + open: mocks.openPictureInPicture, + close: mocks.closePictureInPicture, + }, + }, })); vi.mock("./PreviewChromeRow", () => ({ - PreviewChromeRow: (props: { onSubmit: (url: string) => void }) => { + PreviewChromeRow: (props: { + onSubmit: (url: string) => void; + onPictureInPicture?: () => void; + pictureInPicture?: boolean; + trailingActions?: { + props: { onNativePictureInPicture?: () => void }; + }; + }) => { mocks.submittedUrl = props.onSubmit; + mocks.togglePictureInPicture = props.onPictureInPicture ?? null; + mocks.toggleNativePictureInPicture = + props.trailingActions?.props.onNativePictureInPicture ?? null; + mocks.pictureInPicturePressed = props.pictureInPicture ?? false; return null; }, })); @@ -111,7 +174,12 @@ vi.mock("./PreviewEmptyState", () => ({ return null; }, })); -vi.mock("./PreviewMoreMenu", () => ({ PreviewMoreMenu: () => null })); +vi.mock("./PreviewMoreMenu", () => ({ + PreviewMoreMenu: (props: { onNativePictureInPicture: () => void }) => { + mocks.toggleNativePictureInPicture = props.onNativePictureInPicture; + return null; + }, +})); vi.mock("./PreviewUnreachable", () => ({ PreviewUnreachable: () => null })); vi.mock("./ZoomIndicator", () => ({ ZoomIndicator: () => null })); vi.mock("./AgentBrowserCursor", () => ({ AgentBrowserCursor: () => null })); @@ -128,6 +196,16 @@ describe("PreviewView navigation", () => { mocks.readPreparedConnection.mockClear(); mocks.submittedUrl = null; mocks.emptyStateUrl = null; + mocks.togglePictureInPicture = null; + mocks.toggleNativePictureInPicture = null; + mocks.pictureInPicturePressed = false; + mocks.miniPlayerTabId = null; + mocks.openMiniPlayer.mockClear(); + mocks.closeMiniPlayer.mockClear(); + mocks.closeRightPanel.mockClear(); + mocks.openPictureInPicture.mockClear(); + mocks.closePictureInPicture.mockClear(); + mocks.pictureInPicture = false; mocks.showEmptyState = false; }); @@ -192,4 +270,47 @@ describe("PreviewView navigation", () => { "http://172.25.85.75:5173/app?mode=test#top", ); }); + + it("opens and closes a thread-scoped floating preview for the active tab", async () => { + const props = { + threadRef: { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }, + tabId: "tab-1", + visible: true, + } as const; + + renderToStaticMarkup(); + expect(mocks.pictureInPicturePressed).toBe(false); + mocks.togglePictureInPicture?.(); + expect(mocks.openMiniPlayer).toHaveBeenCalledWith(props.threadRef, "tab-1"); + expect(mocks.closeRightPanel).toHaveBeenCalledWith(props.threadRef); + + mocks.miniPlayerTabId = "tab-1"; + renderToStaticMarkup(); + expect(mocks.pictureInPicturePressed).toBe(true); + mocks.togglePictureInPicture?.(); + expect(mocks.closeMiniPlayer).toHaveBeenCalledWith(props.threadRef); + }); + + it("keeps the native preview window as a secondary action", async () => { + const props = { + threadRef: { + environmentId: EnvironmentId.make("environment-1"), + threadId: ThreadId.make("thread-1"), + }, + tabId: "tab-1", + visible: true, + } as const; + + renderToStaticMarkup(); + mocks.toggleNativePictureInPicture?.(); + await vi.waitFor(() => expect(mocks.openPictureInPicture).toHaveBeenCalledWith("tab-1")); + + mocks.pictureInPicture = true; + renderToStaticMarkup(); + mocks.toggleNativePictureInPicture?.(); + await vi.waitFor(() => expect(mocks.closePictureInPicture).toHaveBeenCalledWith("tab-1")); + }); }); diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index daff913a6d5..db72111d1cc 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -22,6 +22,8 @@ import { resolveDiscoveredServerUrl } from "~/browser/browserTargetResolver"; import { useEnvironment, useEnvironmentHttpBaseUrl } from "~/state/environments"; import { previewEnvironment } from "~/state/preview"; import { useAtomCommand } from "~/state/use-atom-command"; +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { useRightPanelStore } from "~/rightPanelStore"; import { previewBridge } from "./previewBridge"; import { subscribePreviewAction } from "./previewActionBus"; @@ -47,7 +49,7 @@ import { AgentBrowserCursor } from "./AgentBrowserCursor"; import { startBrowserRecording, stopBrowserRecording, - useActiveBrowserRecordingTabId, + useActiveBrowserRecordingTabIds, } from "~/browser/browserRecording"; import { stackedThreadToast, toastManager } from "~/components/ui/toast"; @@ -67,10 +69,13 @@ const localApi = typeof window === "undefined" ? null : ensureLocalApi(); export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, visible }: Props) { const [focusUrlNonce, setFocusUrlNonce] = useState(undefined); const [pickActive, setPickActive] = useState(false); - const activeRecordingTabId = useActiveBrowserRecordingTabId(); + const activeRecordingTabIds = useActiveBrowserRecordingTabIds(); const pickActiveRef = useRef(false); const isMountedRef = useRef(true); const previewState = useThreadPreviewState(threadRef); + const miniPlayer = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), + ); const addPreviewAnnotation = useComposerDraftStore((store) => store.addPreviewAnnotation); const addImage = useComposerDraftStore((store) => store.addImage); const environment = useEnvironment(threadRef.environmentId); @@ -227,11 +232,35 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, void localApi.shell.openExternal(url).catch(() => undefined); }, [url]); + const handlePictureInPicture = useCallback(() => { + if (!tabId) return; + if (miniPlayer?.tabId === tabId) { + usePreviewMiniPlayerStore.getState().close(threadRef); + return; + } + usePreviewMiniPlayerStore.getState().open(threadRef, tabId); + useRightPanelStore.getState().close(threadRef); + }, [miniPlayer?.tabId, tabId, threadRef]); + + const handleNativePictureInPicture = useCallback(() => { + if (!previewBridge || !tabId) return; + const operation = desktopOverlay?.pictureInPicture + ? previewBridge.pictureInPicture.close + : previewBridge.pictureInPicture.open; + void operation(tabId).catch((error) => { + toastManager.add({ + type: "error", + title: "Unable to update popped-out preview", + description: error instanceof Error ? error.message : "An error occurred.", + }); + }); + }, [desktopOverlay?.pictureInPicture, tabId]); + const handleCapture = useCallback( (record: boolean) => { if (!previewBridge || !tabId) return; const bridge = previewBridge; - const recordingThisTab = activeRecordingTabId === tabId; + const recordingThisTab = activeRecordingTabIds.has(tabId); if (recordingThisTab) { void stopBrowserRecording(tabId).then( (artifact) => { @@ -325,15 +354,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, return; } if (record) { - if (activeRecordingTabId !== null) { - toastManager.add({ - type: "warning", - title: "Another preview is recording", - description: "Stop the active recording before starting a new one.", - }); - return; - } - void startBrowserRecording(tabId).catch((error) => { + void startBrowserRecording(tabId, threadRef).catch((error) => { toastManager.add({ type: "error", title: "Unable to start recording", @@ -472,7 +493,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, }, ); }, - [activeRecordingTabId, tabId], + [activeRecordingTabIds, tabId, threadRef], ); const handlePickElement = useCallback(() => { @@ -594,7 +615,10 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, onOpenInBrowser={tabId ? handleOpenInBrowser : undefined} onCapture={previewBridge && tabId ? handleCapture : undefined} captureDisabled={!desktopOverlay || isUnreachable} - recording={tabId !== null && activeRecordingTabId === tabId} + recording={tabId !== null && activeRecordingTabIds.has(tabId)} + onPictureInPicture={previewBridge && tabId ? handlePictureInPicture : undefined} + pictureInPicture={miniPlayer?.tabId === tabId} + pictureInPictureDisabled={!desktopOverlay?.hasWebContents || isUnreachable} onPickElement={previewBridge && tabId ? handlePickElement : undefined} pickActive={pickActive} // Disable when there's no tab (nothing to pick on) OR the page @@ -608,11 +632,13 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, previewBridge ? ( ) : null } diff --git a/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx new file mode 100644 index 00000000000..3f1b5dbf4b7 --- /dev/null +++ b/apps/web/src/components/preview/ThreadPreviewMiniPlayer.tsx @@ -0,0 +1,313 @@ +"use client"; + +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { PanelRightIcon, PictureInPicture2, XIcon } from "lucide-react"; +import { type PointerEvent as ReactPointerEvent, useLayoutEffect, useRef } from "react"; + +import { BrowserSurfaceSlot } from "~/browser/BrowserSurfaceSlot"; +import { Button } from "~/components/ui/button"; +import { toastManager } from "~/components/ui/toast"; +import { useThreadPreviewState } from "~/previewStateStore"; +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; +import { useRightPanelStore } from "~/rightPanelStore"; + +import { previewBridge } from "./previewBridge"; +import { + clampPreviewMiniPlayerPosition, + clampPreviewMiniPlayerSize, + PREVIEW_MINI_PLAYER_DEFAULT_SIZE, +} from "./previewMiniPlayerLayout"; + +interface DragState { + readonly pointerId: number; + readonly pointerX: number; + readonly pointerY: number; + readonly playerX: number; + readonly playerY: number; +} + +interface ResizeState { + readonly pointerId: number; + readonly pointerX: number; + readonly pointerY: number; + readonly width: number; + readonly height: number; +} + +interface Props { + readonly threadRef: ScopedThreadRef; + readonly tabId: string; + readonly bottomInset: number; +} + +export function ThreadPreviewMiniPlayer({ threadRef, tabId, bottomInset }: Props) { + const rootRef = useRef(null); + const dragRef = useRef(null); + const resizeRef = useRef(null); + const miniPlayer = usePreviewMiniPlayerStore((state) => + selectThreadPreviewMiniPlayer(state.byThreadKey, threadRef), + ); + const previewState = useThreadPreviewState(threadRef); + const snapshot = previewState.sessions[tabId] ?? null; + const desktopOverlay = previewState.desktopByTabId[tabId] ?? null; + const position = miniPlayer?.tabId === tabId ? miniPlayer.position : null; + const size = + miniPlayer?.tabId === tabId && miniPlayer.size + ? miniPlayer.size + : PREVIEW_MINI_PLAYER_DEFAULT_SIZE; + const close = () => { + usePreviewMiniPlayerStore.getState().close(threadRef); + }; + + const openInPanel = () => { + usePreviewMiniPlayerStore.getState().close(threadRef); + useRightPanelStore.getState().openBrowser(threadRef, tabId); + }; + + const toggleNativePictureInPicture = () => { + if (!previewBridge) return; + const operation = desktopOverlay?.pictureInPicture + ? previewBridge.pictureInPicture.close + : previewBridge.pictureInPicture.open; + void operation(tabId).catch((error) => { + toastManager.add({ + type: "error", + title: "Unable to update popped-out preview", + description: error instanceof Error ? error.message : "An error occurred.", + }); + }); + }; + + useLayoutEffect(() => { + const clampAndMove = () => { + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement)) return; + const nextSize = clampPreviewMiniPlayerSize( + { width: root.offsetWidth, height: root.offsetHeight }, + { width: parent.clientWidth, height: parent.clientHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().resize(threadRef, tabId, nextSize); + const next = clampPreviewMiniPlayerPosition( + position ?? { x: root.offsetLeft, y: root.offsetTop }, + { width: parent.clientWidth, height: parent.clientHeight }, + nextSize, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); + }; + clampAndMove(); + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement) || typeof ResizeObserver === "undefined") { + return; + } + const observer = new ResizeObserver(clampAndMove); + observer.observe(root); + observer.observe(parent); + return () => observer.disconnect(); + }, [bottomInset, position, tabId, threadRef]); + + const handlePointerDown = (event: ReactPointerEvent) => { + if (event.button !== 0) return; + const root = rootRef.current; + const parent = root?.offsetParent; + if (!root || !(parent instanceof HTMLElement)) return; + const rootRect = root.getBoundingClientRect(); + const parentRect = parent.getBoundingClientRect(); + dragRef.current = { + pointerId: event.pointerId, + pointerX: event.clientX, + pointerY: event.clientY, + playerX: rootRect.left - parentRect.left, + playerY: rootRect.top - parentRect.top, + }; + event.currentTarget.setPointerCapture(event.pointerId); + event.preventDefault(); + }; + + const handlePointerMove = (event: ReactPointerEvent) => { + const drag = dragRef.current; + const root = rootRef.current; + const parent = root?.offsetParent; + if (!drag || drag.pointerId !== event.pointerId || !root || !(parent instanceof HTMLElement)) { + return; + } + const next = clampPreviewMiniPlayerPosition( + { + x: drag.playerX + event.clientX - drag.pointerX, + y: drag.playerY + event.clientY - drag.pointerY, + }, + { width: parent.clientWidth, height: parent.clientHeight }, + { width: root.offsetWidth, height: root.offsetHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, next); + }; + + const endDrag = (event: ReactPointerEvent) => { + if (dragRef.current?.pointerId !== event.pointerId) return; + dragRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + }; + + const handleResizePointerDown = (event: ReactPointerEvent) => { + if (event.button !== 0) return; + const root = rootRef.current; + if (!root) return; + resizeRef.current = { + pointerId: event.pointerId, + pointerX: event.clientX, + pointerY: event.clientY, + width: root.offsetWidth, + height: root.offsetHeight, + }; + event.currentTarget.setPointerCapture(event.pointerId); + event.preventDefault(); + event.stopPropagation(); + }; + + const handleResizePointerMove = (event: ReactPointerEvent) => { + const resize = resizeRef.current; + const root = rootRef.current; + const parent = root?.offsetParent; + if ( + !resize || + resize.pointerId !== event.pointerId || + !root || + !(parent instanceof HTMLElement) + ) { + return; + } + const nextSize = clampPreviewMiniPlayerSize( + { + width: resize.width + event.clientX - resize.pointerX, + height: resize.height + event.clientY - resize.pointerY, + }, + { width: parent.clientWidth, height: parent.clientHeight }, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().resize(threadRef, tabId, nextSize); + const nextPosition = clampPreviewMiniPlayerPosition( + position ?? { x: root.offsetLeft, y: root.offsetTop }, + { width: parent.clientWidth, height: parent.clientHeight }, + nextSize, + bottomInset, + ); + usePreviewMiniPlayerStore.getState().move(threadRef, tabId, nextPosition); + }; + + const endResize = (event: ReactPointerEvent) => { + if (resizeRef.current?.pointerId !== event.pointerId) return; + resizeRef.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + }; + + if (!snapshot || miniPlayer?.tabId !== tabId) return null; + + return ( +
+
+ + +
+
+ +
+ {!desktopOverlay?.hasWebContents ? ( +
+ Reconnecting preview… +
+ ) : null} +
+
+ ); +} diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts index 90de86f799d..a199eded323 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.test.ts @@ -1,7 +1,10 @@ import type { PreviewAutomationOpenInput, PreviewSessionSnapshot } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { previewAutomationOpenNeedsOverlay } from "./previewAutomationOpenReadiness"; +import { + previewAutomationOpenNeedsOverlay, + shouldOpenPreviewMiniPlayer, +} from "./previewAutomationOpenReadiness"; const snapshot = (navStatus: PreviewSessionSnapshot["navStatus"]): PreviewSessionSnapshot => ({ threadId: "thread-1", @@ -13,6 +16,18 @@ const snapshot = (navStatus: PreviewSessionSnapshot["navStatus"]): PreviewSessio }); describe("preview automation open readiness", () => { + it("opens the inline preview by default", () => { + expect(shouldOpenPreviewMiniPlayer({} as PreviewAutomationOpenInput)).toBe(true); + }); + + it("supports explicit opt-out and the legacy show alias", () => { + expect(shouldOpenPreviewMiniPlayer({ open: false } as PreviewAutomationOpenInput)).toBe(false); + expect(shouldOpenPreviewMiniPlayer({ show: false } as PreviewAutomationOpenInput)).toBe(false); + expect( + shouldOpenPreviewMiniPlayer({ open: true, show: false } as PreviewAutomationOpenInput), + ).toBe(true); + }); + it("does not wait for a desktop overlay when opening an empty tab", () => { expect( previewAutomationOpenNeedsOverlay( diff --git a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts index 416c2f87c64..1f78c22621f 100644 --- a/apps/web/src/components/preview/previewAutomationOpenReadiness.ts +++ b/apps/web/src/components/preview/previewAutomationOpenReadiness.ts @@ -1,5 +1,9 @@ import type { PreviewAutomationOpenInput, PreviewSessionSnapshot } from "@t3tools/contracts"; +export function shouldOpenPreviewMiniPlayer(input: PreviewAutomationOpenInput): boolean { + return input.open ?? input.show ?? true; +} + export function previewAutomationOpenNeedsOverlay( input: PreviewAutomationOpenInput, snapshot: PreviewSessionSnapshot, diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts new file mode 100644 index 00000000000..98b26e8a0b2 --- /dev/null +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + clampPreviewMiniPlayerPosition, + clampPreviewMiniPlayerSize, + PREVIEW_MINI_PLAYER_EDGE_GAP, +} from "./previewMiniPlayerLayout"; + +describe("clampPreviewMiniPlayerPosition", () => { + it("keeps a dragged player within the chat viewport", () => { + expect( + clampPreviewMiniPlayerPosition( + { x: 900, y: -40 }, + { width: 1_000, height: 700 }, + { width: 360, height: 240 }, + ), + ).toEqual({ + x: 628, + y: PREVIEW_MINI_PLAYER_EDGE_GAP, + }); + }); + + it("keeps an edge gap when the player is larger than its container", () => { + expect( + clampPreviewMiniPlayerPosition( + { x: 20, y: 30 }, + { width: 200, height: 160 }, + { width: 360, height: 240 }, + ), + ).toEqual({ + x: PREVIEW_MINI_PLAYER_EDGE_GAP, + y: PREVIEW_MINI_PLAYER_EDGE_GAP, + }); + }); + + it("keeps the player above a growing composer inset", () => { + expect( + clampPreviewMiniPlayerPosition( + { x: 500, y: 448 }, + { width: 1_000, height: 700 }, + { width: 360, height: 240 }, + 160, + ), + ).toEqual({ + x: 500, + y: 288, + }); + }); +}); + +describe("clampPreviewMiniPlayerSize", () => { + it("allows resizing within the available chat viewport", () => { + expect( + clampPreviewMiniPlayerSize({ width: 520, height: 360 }, { width: 1_000, height: 700 }, 120), + ).toEqual({ width: 520, height: 360 }); + }); + + it("bounds oversized players above the composer", () => { + expect( + clampPreviewMiniPlayerSize( + { width: 2_000, height: 2_000 }, + { width: 1_000, height: 700 }, + 120, + ), + ).toEqual({ width: 976, height: 556 }); + }); + + it("lets a tiny container win over the preferred minimum", () => { + expect( + clampPreviewMiniPlayerSize({ width: 360, height: 239 }, { width: 250, height: 180 }, 20), + ).toEqual({ width: 226, height: 136 }); + }); +}); diff --git a/apps/web/src/components/preview/previewMiniPlayerLayout.ts b/apps/web/src/components/preview/previewMiniPlayerLayout.ts new file mode 100644 index 00000000000..abdd5b8ca4d --- /dev/null +++ b/apps/web/src/components/preview/previewMiniPlayerLayout.ts @@ -0,0 +1,46 @@ +import type { PreviewMiniPlayerPosition, PreviewMiniPlayerSize } from "~/previewMiniPlayerStore"; + +export const PREVIEW_MINI_PLAYER_EDGE_GAP = 12; +export const PREVIEW_MINI_PLAYER_DEFAULT_SIZE = { width: 320, height: 200 } as const; +export const PREVIEW_MINI_PLAYER_MIN_SIZE = { width: 240, height: 150 } as const; + +export function clampPreviewMiniPlayerSize( + size: PreviewMiniPlayerSize, + container: PreviewMiniPlayerSize, + bottomInset = 0, +): PreviewMiniPlayerSize { + const availableWidth = Math.max(1, container.width - PREVIEW_MINI_PLAYER_EDGE_GAP * 2); + const availableHeight = Math.max( + 1, + container.height - Math.max(0, bottomInset) - PREVIEW_MINI_PLAYER_EDGE_GAP * 2, + ); + return { + width: Math.round( + Math.min(Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.width, size.width), availableWidth), + ), + height: Math.round( + Math.min(Math.max(PREVIEW_MINI_PLAYER_MIN_SIZE.height, size.height), availableHeight), + ), + }; +} + +export function clampPreviewMiniPlayerPosition( + position: PreviewMiniPlayerPosition, + container: PreviewMiniPlayerSize, + player: PreviewMiniPlayerSize, + bottomInset = 0, +): PreviewMiniPlayerPosition { + const reservedBottomSpace = Math.max(0, bottomInset); + const maxX = Math.max( + PREVIEW_MINI_PLAYER_EDGE_GAP, + container.width - player.width - PREVIEW_MINI_PLAYER_EDGE_GAP, + ); + const maxY = Math.max( + PREVIEW_MINI_PLAYER_EDGE_GAP, + container.height - reservedBottomSpace - player.height - PREVIEW_MINI_PLAYER_EDGE_GAP, + ); + return { + x: Math.min(Math.max(position.x, PREVIEW_MINI_PLAYER_EDGE_GAP), maxX), + y: Math.min(Math.max(position.y, PREVIEW_MINI_PLAYER_EDGE_GAP), maxY), + }; +} diff --git a/apps/web/src/components/preview/usePreviewBridge.ts b/apps/web/src/components/preview/usePreviewBridge.ts index 22d17f968ba..4e7fccd3f2e 100644 --- a/apps/web/src/components/preview/usePreviewBridge.ts +++ b/apps/web/src/components/preview/usePreviewBridge.ts @@ -73,10 +73,12 @@ function shouldClearBrowserPointer( function projectDesktopState(state: DesktopPreviewTabState): DesktopPreviewOverlay { return { + hasWebContents: state.webContentsId !== null, canGoBack: state.canGoBack, canGoForward: state.canGoForward, loading: state.navStatus.kind === "Loading", zoomFactor: state.zoomFactor, + pictureInPicture: state.pictureInPicture, colorScheme: state.colorScheme, controller: state.controller, }; diff --git a/apps/web/src/components/preview/usePreviewSession.ts b/apps/web/src/components/preview/usePreviewSession.ts index 9bc2cc84c37..b2865c4eac1 100644 --- a/apps/web/src/components/preview/usePreviewSession.ts +++ b/apps/web/src/components/preview/usePreviewSession.ts @@ -2,14 +2,12 @@ import { useAtomValue } from "@effect/atom-react"; import { parseScopedThreadKey, scopedThreadKey } from "@t3tools/client-runtime/environment"; -import { runAtomCommand } from "@t3tools/client-runtime/state/runtime"; import type { ScopedThreadRef } from "@t3tools/contracts"; import * as Schema from "effect/Schema"; import { AsyncResult, Atom } from "effect/unstable/reactivity"; import { applyPreviewServerEvent, - applyPreviewServerSnapshot, readThreadPreviewState, reconcilePreviewServerSessions, } from "~/previewStateStore"; @@ -41,61 +39,26 @@ const previewSessionSyncAtom = Atom.family((threadKey: string) => { return Atom.make((get) => { let disposed = false; - let recoveryId = 0; - let recoveringUrl: string | null = null; let sessionsVersion = 0; let eventsVersion = 0; const reconcileSessions = (result: Atom.Type) => { if (!AsyncResult.isSuccess(result)) return; - if (result.value.sessions.length > 0) { - recoveringUrl = null; - recoveryId += 1; - reconcilePreviewServerSessions(threadRef, result.value.sessions); - return; - } - - const localSnapshot = readThreadPreviewState(threadRef).snapshot; - const recoverableUrl = - localSnapshot && localSnapshot.navStatus._tag !== "Idle" - ? localSnapshot.navStatus.url - : null; - if (!recoverableUrl) { - applyPreviewServerSnapshot(threadRef, null); - return; - } - if (recoveringUrl === recoverableUrl) return; - - recoveringUrl = recoverableUrl; - const currentRecoveryId = ++recoveryId; - void runAtomCommand( - get.registry, - previewEnvironment.open, - { - environmentId: threadRef.environmentId, - input: { threadId: threadRef.threadId, url: recoverableUrl }, - }, - { reportDefect: false, reportFailure: false }, - ).then((openResult) => { - if (disposed || currentRecoveryId !== recoveryId) return; - recoveringUrl = null; - if (openResult._tag === "Failure") return; - applyPreviewServerSnapshot(threadRef, openResult.value); - get.refresh(sessionsAtom); - }); + reconcilePreviewServerSessions(threadRef, result.value); }; const applyLatestEvent = (result: Atom.Type) => { if (!AsyncResult.isSuccess(result) || result.value.threadId !== threadRef.threadId) return; - applyPreviewServerEvent(threadRef, result.value); - if (result.value.type === "opened" || result.value.type === "closed") { + const currentEpoch = readThreadPreviewState(threadRef).serverEpoch; + if (currentEpoch !== null && currentEpoch !== result.value.serverEpoch) { get.refresh(sessionsAtom); + return; } + applyPreviewServerEvent(threadRef, result.value); }; get.addFinalizer(() => { disposed = true; - recoveryId += 1; }); const initialSessions = get.once(sessionsAtom); const initialEvent = get.once(eventsAtom); diff --git a/apps/web/src/previewMiniPlayerStore.test.ts b/apps/web/src/previewMiniPlayerStore.test.ts new file mode 100644 index 00000000000..d6ec64bf069 --- /dev/null +++ b/apps/web/src/previewMiniPlayerStore.test.ts @@ -0,0 +1,64 @@ +import { scopeThreadRef } from "@t3tools/client-runtime/environment"; +import { type EnvironmentId, ThreadId } from "@t3tools/contracts"; +import { beforeEach, describe, expect, it } from "vite-plus/test"; + +import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "./previewMiniPlayerStore"; + +const refA = scopeThreadRef("env-1" as EnvironmentId, ThreadId.make("thread-A")); +const refB = scopeThreadRef("env-1" as EnvironmentId, ThreadId.make("thread-B")); + +beforeEach(() => { + usePreviewMiniPlayerStore.setState({ byThreadKey: {} }); +}); + +describe("previewMiniPlayerStore", () => { + it("keeps floating previews scoped to their thread", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().open(refB, "tab-b"); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toMatchObject({ tabId: "tab-a" }); + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refB), + ).toMatchObject({ tabId: "tab-b" }); + }); + + it("preserves position when switching the floating tab within one thread", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().move(refA, "tab-a", { x: 24, y: 48 }); + usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toEqual({ + tabId: "tab-b", + position: { x: 24, y: 48 }, + size: null, + }); + }); + + it("ignores stale drag updates after the floating tab changes", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + usePreviewMiniPlayerStore.getState().move(refA, "tab-a", { x: 100, y: 100 }); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toEqual({ + tabId: "tab-b", + position: null, + size: null, + }); + }); + + it("preserves a thread-bound size while switching tabs", () => { + usePreviewMiniPlayerStore.getState().open(refA, "tab-a"); + usePreviewMiniPlayerStore.getState().resize(refA, "tab-a", { width: 480, height: 320 }); + usePreviewMiniPlayerStore.getState().open(refA, "tab-b"); + + expect( + selectThreadPreviewMiniPlayer(usePreviewMiniPlayerStore.getState().byThreadKey, refA), + ).toMatchObject({ tabId: "tab-b", size: { width: 480, height: 320 } }); + }); +}); diff --git a/apps/web/src/previewMiniPlayerStore.ts b/apps/web/src/previewMiniPlayerStore.ts new file mode 100644 index 00000000000..d1a1fde5eff --- /dev/null +++ b/apps/web/src/previewMiniPlayerStore.ts @@ -0,0 +1,96 @@ +import { scopedThreadKey } from "@t3tools/client-runtime/environment"; +import type { ScopedThreadRef } from "@t3tools/contracts"; +import { create } from "zustand"; + +export interface PreviewMiniPlayerPosition { + readonly x: number; + readonly y: number; +} + +export interface PreviewMiniPlayerSize { + readonly width: number; + readonly height: number; +} + +export interface PreviewMiniPlayerState { + readonly tabId: string; + readonly position: PreviewMiniPlayerPosition | null; + readonly size: PreviewMiniPlayerSize | null; +} + +interface PreviewMiniPlayerStoreState { + readonly byThreadKey: Record; + readonly open: (ref: ScopedThreadRef, tabId: string) => void; + readonly close: (ref: ScopedThreadRef) => void; + readonly move: (ref: ScopedThreadRef, tabId: string, position: PreviewMiniPlayerPosition) => void; + readonly resize: (ref: ScopedThreadRef, tabId: string, size: PreviewMiniPlayerSize) => void; + readonly removeThread: (ref: ScopedThreadRef) => void; +} + +export const usePreviewMiniPlayerStore = create()((set) => ({ + byThreadKey: {}, + open: (ref, tabId) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const current = state.byThreadKey[threadKey]; + if (current?.tabId === tabId) return state; + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { + tabId, + position: current?.position ?? null, + size: current?.size ?? null, + }, + }, + }; + }), + close: (ref) => + set((state) => { + const threadKey = scopedThreadKey(ref); + if (!(threadKey in state.byThreadKey)) return state; + const { [threadKey]: _closed, ...byThreadKey } = state.byThreadKey; + return { byThreadKey }; + }), + move: (ref, tabId, position) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const current = state.byThreadKey[threadKey]; + if (!current || current.tabId !== tabId) return state; + if (current.position?.x === position.x && current.position.y === position.y) return state; + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { ...current, position }, + }, + }; + }), + resize: (ref, tabId, size) => + set((state) => { + const threadKey = scopedThreadKey(ref); + const current = state.byThreadKey[threadKey]; + if (!current || current.tabId !== tabId) return state; + if (current.size?.width === size.width && current.size.height === size.height) return state; + return { + byThreadKey: { + ...state.byThreadKey, + [threadKey]: { ...current, size }, + }, + }; + }), + removeThread: (ref) => + set((state) => { + const threadKey = scopedThreadKey(ref); + if (!(threadKey in state.byThreadKey)) return state; + const { [threadKey]: _removed, ...byThreadKey } = state.byThreadKey; + return { byThreadKey }; + }), +})); + +export function selectThreadPreviewMiniPlayer( + byThreadKey: Record, + ref: ScopedThreadRef | null | undefined, +): PreviewMiniPlayerState | null { + if (!ref) return null; + return byThreadKey[scopedThreadKey(ref)] ?? null; +} diff --git a/apps/web/src/previewStateStore.test.ts b/apps/web/src/previewStateStore.test.ts index f4a3eb732ba..a4d709e2f29 100644 --- a/apps/web/src/previewStateStore.test.ts +++ b/apps/web/src/previewStateStore.test.ts @@ -1,11 +1,16 @@ import { scopedThreadKey, scopeThreadRef } from "@t3tools/client-runtime/environment"; -import { type EnvironmentId, type PreviewSessionSnapshot, ThreadId } from "@t3tools/contracts"; +import { + type EnvironmentId, + type PreviewEvent, + type PreviewSessionSnapshot, + ThreadId, +} from "@t3tools/contracts"; import { beforeEach, describe, expect, it } from "vite-plus/test"; import { __testing, applyPreviewDesktopState, - applyPreviewServerEvent, + applyPreviewServerEvent as applyPreviewServerEventImpl, applyPreviewServerSnapshot, beginPreviewSessionClose, cancelPreviewSessionClose, @@ -33,7 +38,25 @@ const makeSnapshot = (overrides: Partial = {}): PreviewS ...overrides, }); +type PreviewEventDraft = PreviewEvent extends infer Event + ? Event extends { readonly revision: number } + ? Omit + : never + : never; + +const serverEpoch = "server-a"; +let nextServerRevision = 0; +const applyPreviewServerEvent = (eventRef: typeof ref, event: PreviewEventDraft): void => { + nextServerRevision += 1; + applyPreviewServerEventImpl(eventRef, { + ...event, + serverEpoch, + revision: nextServerRevision, + } as PreviewEvent); +}; + beforeEach(() => { + nextServerRevision = 0; resetPreviewStateForTests(); }); @@ -291,10 +314,12 @@ describe("previewStateStore (single-tab)", () => { snapshot, }); applyPreviewDesktopState(ref, snapshot.tabId, { + hasWebContents: true, canGoBack: true, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: false, colorScheme: "system", controller: "none", }); @@ -309,10 +334,12 @@ describe("previewStateStore (single-tab)", () => { applyPreviewServerSnapshot(ref, first); applyPreviewServerSnapshot(ref, second); applyPreviewDesktopState(ref, first.tabId, { + hasWebContents: true, canGoBack: true, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: false, colorScheme: "system", controller: "none", }); @@ -355,15 +382,17 @@ describe("previewStateStore (single-tab)", () => { applyPreviewServerSnapshot(ref, stale); applyPreviewServerSnapshot(ref, active); applyPreviewDesktopState(ref, stale.tabId, { + hasWebContents: true, canGoBack: false, canGoForward: false, loading: false, zoomFactor: 1, + pictureInPicture: false, colorScheme: "system", controller: "none", }); - reconcilePreviewServerSessions(ref, [active]); + reconcilePreviewServerSessions(ref, { sessions: [active], serverEpoch, revision: 1 }); const state = readThreadPreviewState(ref); expect(Object.keys(state.sessions)).toEqual([active.tabId]); @@ -375,7 +404,7 @@ describe("previewStateStore (single-tab)", () => { it("clears stale sessions when an authoritative list is empty", () => { applyPreviewServerSnapshot(ref, makeSnapshot()); - reconcilePreviewServerSessions(ref, []); + reconcilePreviewServerSessions(ref, { sessions: [], serverEpoch, revision: 1 }); const state = readThreadPreviewState(ref); expect(state.sessions).toEqual({}); @@ -383,6 +412,74 @@ describe("previewStateStore (single-tab)", () => { expect(state.snapshot).toBeNull(); }); + it("ignores a list response older than the latest server event", () => { + const snapshot = makeSnapshot(); + applyPreviewServerEvent(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + snapshot, + }); + + reconcilePreviewServerSessions(ref, { sessions: [], serverEpoch, revision: 0 }); + + expect(readThreadPreviewState(ref).sessions).toEqual({ [snapshot.tabId]: snapshot }); + }); + + it("does not resurrect a tab from an event older than its close", () => { + const snapshot = makeSnapshot(); + applyPreviewServerEvent(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + snapshot, + }); + applyPreviewServerEvent(ref, { + type: "closed", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: "2026-01-01T00:00:01.000Z", + }); + + applyPreviewServerEventImpl(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + serverEpoch, + revision: 1, + snapshot, + }); + + expect(readThreadPreviewState(ref).sessions).toEqual({}); + }); + + it("accepts a lower revision from a newly restarted server", () => { + const snapshot = makeSnapshot(); + applyPreviewServerEventImpl(ref, { + type: "opened", + threadId: "thread-1", + tabId: snapshot.tabId, + createdAt: snapshot.updatedAt, + serverEpoch, + revision: 12, + snapshot, + }); + + reconcilePreviewServerSessions(ref, { + sessions: [], + serverEpoch: "server-b", + revision: 0, + }); + + const state = readThreadPreviewState(ref); + expect(state.sessions).toEqual({}); + expect(state.serverEpoch).toBe("server-b"); + expect(state.serverRevision).toBe(0); + }); + it("applyServerSnapshot null clears snapshot for a thread that had one", () => { const snapshot = makeSnapshot(); applyPreviewServerSnapshot(ref, snapshot); diff --git a/apps/web/src/previewStateStore.ts b/apps/web/src/previewStateStore.ts index 34553d6f80b..02b66922a43 100644 --- a/apps/web/src/previewStateStore.ts +++ b/apps/web/src/previewStateStore.ts @@ -10,6 +10,7 @@ import { scopedThreadKey } from "@t3tools/client-runtime/environment"; import { type DesktopPreviewColorScheme, type PreviewEvent, + type PreviewListResult, type PreviewSessionSnapshot, type ScopedThreadRef, } from "@t3tools/contracts"; @@ -19,10 +20,12 @@ import { PREVIEW_RECENT_URL_LIMIT } from "./components/preview/previewConstants" import { appAtomRegistry } from "./rpc/atomRegistry"; export interface DesktopPreviewOverlay { + hasWebContents: boolean; canGoBack: boolean; canGoForward: boolean; loading: boolean; zoomFactor: number; + pictureInPicture: boolean; colorScheme: DesktopPreviewColorScheme; controller: "human" | "agent" | "none"; } @@ -36,6 +39,10 @@ export interface ThreadPreviewState { desktopOverlay: DesktopPreviewOverlay | null; desktopByTabId: Record; recentlySeenUrls: string[]; + /** Server process currently authoritative for revision ordering. */ + serverEpoch: string | null; + /** Latest ordered server revision applied from a list response or event. */ + serverRevision: number; } const EMPTY_THREAD_PREVIEW_STATE: ThreadPreviewState = Object.freeze({ @@ -46,6 +53,8 @@ const EMPTY_THREAD_PREVIEW_STATE: ThreadPreviewState = Object.freeze({ desktopOverlay: null, desktopByTabId: {}, recentlySeenUrls: [] as string[], + serverEpoch: null, + serverRevision: 0, }); const emptyPreviewStateAtom = Atom.make(EMPTY_THREAD_PREVIEW_STATE).pipe( @@ -175,52 +184,68 @@ export function subscribeThreadPreviewState( export function applyPreviewServerEvent(ref: ScopedThreadRef, event: PreviewEvent): void { updateThreadPreviewState(ref, (current) => { - switch (event.type) { - case "opened": - case "navigated": - case "resized": { - const snapshot = event.snapshot; - if (current.suppressedTabIds.has(snapshot.tabId)) return current; - const recentlySeenUrls = - snapshot.navStatus._tag === "Idle" - ? current.recentlySeenUrls - : dedupeRecentUrls(current.recentlySeenUrls, snapshot.navStatus.url); - const sessions = { ...current.sessions, [snapshot.tabId]: snapshot }; - const activeTabId = event.type === "opened" ? snapshot.tabId : current.activeTabId; - const activeSnapshot = sessions[activeTabId ?? snapshot.tabId] ?? snapshot; - return { - ...current, - sessions, - activeTabId: activeTabId ?? snapshot.tabId, - snapshot: activeSnapshot, - desktopOverlay: current.desktopByTabId[activeSnapshot.tabId] ?? null, - recentlySeenUrls, - }; + if (current.serverEpoch !== null && event.serverEpoch !== current.serverEpoch) return current; + if (event.revision < current.serverRevision) return current; + const next = (() => { + switch (event.type) { + case "opened": + case "navigated": + case "resized": { + const snapshot = event.snapshot; + if (current.suppressedTabIds.has(snapshot.tabId)) return current; + const recentlySeenUrls = + snapshot.navStatus._tag === "Idle" + ? current.recentlySeenUrls + : dedupeRecentUrls(current.recentlySeenUrls, snapshot.navStatus.url); + const sessions = { ...current.sessions, [snapshot.tabId]: snapshot }; + const activeTabId = event.type === "opened" ? snapshot.tabId : current.activeTabId; + const activeSnapshot = sessions[activeTabId ?? snapshot.tabId] ?? snapshot; + return { + ...current, + sessions, + activeTabId: activeTabId ?? snapshot.tabId, + snapshot: activeSnapshot, + desktopOverlay: current.desktopByTabId[activeSnapshot.tabId] ?? null, + recentlySeenUrls, + }; + } + case "failed": { + const existing = current.sessions[event.tabId]; + if (!existing) return current; + const failedSnapshot = { + ...existing, + navStatus: { + _tag: "LoadFailed" as const, + url: event.url, + title: event.title, + code: event.code, + description: event.description, + }, + updatedAt: event.createdAt, + }; + const sessions = { ...current.sessions, [event.tabId]: failedSnapshot }; + return { + ...current, + sessions, + snapshot: current.activeTabId === event.tabId ? failedSnapshot : current.snapshot, + }; + } + case "closed": { + const closed = removeSession(current, event.tabId); + if (!closed.suppressedTabIds.has(event.tabId)) return closed; + const suppressedTabIds = new Set(closed.suppressedTabIds); + suppressedTabIds.delete(event.tabId); + return { ...closed, suppressedTabIds }; + } } - case "failed": { - const existing = current.sessions[event.tabId]; - if (!existing) return current; - const failedSnapshot = { - ...existing, - navStatus: { - _tag: "LoadFailed" as const, - url: event.url, - title: event.title, - code: event.code, - description: event.description, - }, - updatedAt: event.createdAt, - }; - const sessions = { ...current.sessions, [event.tabId]: failedSnapshot }; - return { - ...current, - sessions, - snapshot: current.activeTabId === event.tabId ? failedSnapshot : current.snapshot, + })(); + return next.serverRevision === event.revision && next.serverEpoch === event.serverEpoch + ? next + : { + ...next, + serverEpoch: event.serverEpoch, + serverRevision: event.revision, }; - } - case "closed": - return removeSession(current, event.tabId); - } }); } @@ -291,9 +316,12 @@ export function updatePreviewServerSnapshot( */ export function reconcilePreviewServerSessions( ref: ScopedThreadRef, - snapshots: ReadonlyArray, + result: PreviewListResult, ): void { updateThreadPreviewState(ref, (current) => { + const sameServer = current.serverEpoch === result.serverEpoch; + if (sameServer && result.revision < current.serverRevision) return current; + const snapshots = result.sessions; const sessions: Record = {}; let recentlySeenUrls = current.recentlySeenUrls; for (const snapshot of snapshots) { @@ -313,14 +341,22 @@ export function reconcilePreviewServerSessions( const desktopByTabId = Object.fromEntries( Object.entries(current.desktopByTabId).filter(([tabId]) => sessions[tabId] !== undefined), ); + const suppressedTabIds = new Set( + [...current.suppressedTabIds].filter((tabId) => + snapshots.some((snapshot) => snapshot.tabId === tabId), + ), + ); return { ...current, sessions, + suppressedTabIds, activeTabId, snapshot, desktopByTabId, desktopOverlay: activeTabId ? (desktopByTabId[activeTabId] ?? null) : null, recentlySeenUrls, + serverEpoch: result.serverEpoch, + serverRevision: result.revision, }; }); } diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 4b1676d7926..f6f28262a7c 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -527,6 +527,8 @@ export interface DesktopPreviewTabState { canGoForward: boolean; /** Current zoom factor (1.0 = 100%). */ zoomFactor: number; + /** Whether this tab is currently mirrored into a desktop picture-in-picture window. */ + pictureInPicture: boolean; colorScheme: DesktopPreviewColorScheme; controller: "human" | "agent" | "none"; updatedAt: string; @@ -564,6 +566,7 @@ export const DesktopPreviewTabStateSchema: Schema.Codec canGoBack: Schema.Boolean, canGoForward: Schema.Boolean, zoomFactor: Schema.Number, + pictureInPicture: Schema.Boolean, colorScheme: DesktopPreviewColorSchemeSchema, controller: Schema.Literals(["human", "agent", "none"]), updatedAt: Schema.String, @@ -1080,6 +1083,10 @@ export interface DesktopPreviewBridge { captureScreenshot: (tabId: string) => Promise; revealArtifact: (path: string) => Promise; copyArtifactToClipboard: (path: string) => Promise; + pictureInPicture: { + open: (tabId: string) => Promise; + close: (tabId: string) => Promise; + }; recording: { startScreencast: (tabId: string) => Promise; stopScreencast: (tabId: string) => Promise; diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index e2e8336cda9..09a13cd31da 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -29,6 +29,16 @@ const decodeAutomationHost = Schema.decodeUnknownSync(PreviewAutomationHost); const decodeAutomationError = Schema.decodeUnknownSync(PreviewAutomationError); const decodeAutomationStatus = Schema.decodeUnknownSync(PreviewAutomationStatus); +describe("PreviewAutomationOpenInput", () => { + it("accepts the inline preview visibility flag", () => { + expect(decodeOpenInput({ open: false })).toEqual({ open: false }); + }); + + it("retains the legacy show visibility alias", () => { + expect(decodeOpenInput({ show: false })).toEqual({ show: false }); + }); +}); + describe("PreviewNavStatus", () => { it("decodes Idle", () => { expect(decodeNavStatus({ _tag: "Idle" })).toEqual({ _tag: "Idle" }); @@ -216,6 +226,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, snapshot: { threadId: "t", tabId: "preview-t", @@ -234,6 +246,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, url: "https://example.com/", title: "", code: -105, @@ -251,6 +265,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, snapshot: { threadId: "t", tabId: "preview-t", @@ -270,6 +286,8 @@ describe("PreviewEvent", () => { threadId: "t", tabId: "preview-t", createdAt: "2026-01-01T00:00:00.000Z", + serverEpoch: "server-a", + revision: 1, }); expect(event.type).toBe("closed"); }); diff --git a/packages/contracts/src/preview.ts b/packages/contracts/src/preview.ts index c00f878bcbe..dfc10e0b9b7 100644 --- a/packages/contracts/src/preview.ts +++ b/packages/contracts/src/preview.ts @@ -9,7 +9,7 @@ * @module Preview */ import { Schema } from "effect"; -import { ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; +import { NonNegativeInt, PositiveInt, ThreadId, TrimmedNonEmptyString } from "./baseSchemas.ts"; const Url = TrimmedNonEmptyString.check(Schema.isMaxLength(2048)); const Title = Schema.String.check(Schema.isMaxLength(512)); @@ -192,6 +192,10 @@ export type PreviewListInput = typeof PreviewListInput.Type; export const PreviewListResult = Schema.Struct({ sessions: Schema.Array(PreviewSessionSnapshot), + /** Identifies the current server process so revision resets are safe. */ + serverEpoch: TrimmedNonEmptyString, + /** Monotonic server state revision used to reject stale list responses. */ + revision: NonNegativeInt, }); export type PreviewListResult = typeof PreviewListResult.Type; @@ -199,6 +203,10 @@ const PreviewEventBaseSchema = Schema.Struct({ threadId: TrimmedNonEmptyString, tabId: PreviewTabId, createdAt: Schema.String, + /** Identifies the server process that emitted this event. */ + serverEpoch: TrimmedNonEmptyString, + /** Monotonic server state revision shared with PreviewListResult. */ + revision: PositiveInt, }); const PreviewOpenedEvent = Schema.Struct({ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index f05623cbc99..d89c4d6f66e 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -86,9 +86,16 @@ export const PreviewAutomationOpenInput = Schema.Struct({ description: "Optional initial page URL, for example https://t3.chat or localhost:5173. Omit to open a blank tab.", }), + open: Schema.optional( + Schema.Boolean.annotate({ + description: + "Whether to open the thread-bound inline preview for the human. Defaults to true; set false for background-only automation.", + }), + ), show: Schema.optional( Schema.Boolean.annotate({ - description: "Whether to reveal the preview panel to the human. Defaults to true.", + description: + "Deprecated alias for open. Whether to reveal the thread-bound inline preview to the human.", }), ), reuseExistingTab: Schema.optional(