In-app browser panel with agent control

.github/workflows/ci.yml

@@ -49,6 +49,9 @@ jobs:
       - name: Lint
         run: vp exec vp lint --report-unused-disable-directives
 
+      - name: Vendored Playwright injected script is current
+        run: node scripts/extract-playwright-injected.ts --check
+
       - name: Typecheck
         run: vp exec vp run -r --concurrency-limit 2 typecheck
 

.oxfmtrc.json

@@ -12,6 +12,7 @@
     "packages/effect-codex-app-server/src/_generated/schema.gen.ts",
     "apps/web/public/mockServiceWorker.js",
     "apps/web/src/lib/vendor/qrcodegen.ts",
+    "apps/desktop/src/preview/vendor/**",
     "*.icon/**"
   ],
   "sortPackageJson": {},

apps/desktop/package.json

@@ -6,15 +6,16 @@
   "type": "module",
   "main": "dist-electron/main.cjs",
   "scripts": {
+    "build": "vp pack && node scripts/preview-vendor.mjs",
     "dev": "node scripts/dev.mjs",
-    "dev:server-bundle": "vp run --filter @threadlines/server dev:bundle",
     "dev:bundle": "vp pack --watch",
     "dev:electron": "node scripts/dev-electron.mjs",
-    "build": "vp pack",
+    "dev:server-bundle": "vp run --filter @threadlines/server dev:bundle",
+    "smoke-test": "node scripts/smoke-test.mjs",
     "start": "node scripts/start-electron.mjs",
-    "typecheck": "tsc --noEmit",
     "test": "vp test run --passWithNoTests",
-    "smoke-test": "node scripts/smoke-test.mjs"
+    "typecheck": "tsc --noEmit",
+    "vendor:playwright": "node ../../scripts/extract-playwright-injected.ts"
   },
   "dependencies": {
     "@effect/platform-node": "catalog:",
@@ -30,6 +31,7 @@
     "@effect/vitest": "catalog:",
     "@types/node": "catalog:",
     "electron-builder": "^26.15.3",
+    "playwright-core": "^1.61.1",
     "typescript": "catalog:",
     "vite": "catalog:",
     "vite-plus": "catalog:",

apps/desktop/scripts/dev-electron.mjs

@@ -3,6 +3,7 @@ import { unwatchFile, watch, watchFile } from "node:fs";
 import { join } from "node:path";
 
 import { desktopDir, resolveElectronPath } from "./electron-launcher.mjs";
+import { copyPreviewVendor } from "./preview-vendor.mjs";
 import { waitForResources } from "./wait-for-resources.mjs";
 
 const devServerUrl = process.env.VITE_DEV_SERVER_URL?.trim();
@@ -182,6 +183,11 @@ function startApp() {
     return;
   }
 
+  // Every launch, not once at startup: `vp pack --watch` clears its output
+  // directory, and a restart triggered by that rebuild would otherwise start an
+  // Electron whose vendored assets had just been deleted.
+  copyPreviewVendor();
+
   const launchArgs = [
     "--threadlines-dev-root=" + devProcessIdentity,
     ...(desktopUserDataDirectory === null ? [] : ["--user-data-dir=" + desktopUserDataDirectory]),

apps/desktop/scripts/preview-vendor.mjs

@@ -0,0 +1,48 @@
+/**
+ * Puts the vendored Playwright script next to the bundle that reads it.
+ *
+ * `vp pack` bundles code and nothing else, so a static asset under `src/` never
+ * reaches `dist-electron/`. The bundled main resolves the script relative to
+ * itself -- correct in a packaged app, where `src/` does not exist -- so without
+ * this copy the file is only ever where the runtime is not looking.
+ *
+ * The symptom is quiet, which is the reason this exists as its own step rather
+ * than a line in a build script: `browser_snapshot` fails with ENOENT, the model
+ * shrugs and falls back to screenshots, and the agent still looks like it works.
+ */
+import { copyFileSync, existsSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import {
+  VENDOR_DIRECTORY_NAME,
+  VENDORED_INJECTED_MANIFEST_FILENAME,
+  VENDORED_INJECTED_SCRIPT_FILENAME,
+} from "@threadlines/shared/previewInjectedScript";
+
+const desktopDir = dirname(dirname(fileURLToPath(import.meta.url)));
+
+/** The manifest travels with the script so a shipped build can say which
+ *  playwright-core it came from. */
+const assets = [VENDORED_INJECTED_SCRIPT_FILENAME, VENDORED_INJECTED_MANIFEST_FILENAME];
+
+export function copyPreviewVendor() {
+  const from = join(desktopDir, "src", "preview", VENDOR_DIRECTORY_NAME);
+  const to = join(desktopDir, "dist-electron", VENDOR_DIRECTORY_NAME);
+
+  mkdirSync(to, { recursive: true });
+  for (const asset of assets) {
+    const source = join(from, asset);
+    if (!existsSync(source)) {
+      throw new Error(
+        `Missing vendored preview asset ${source}. ` +
+          `Run 'vp run --filter @threadlines/desktop vendor:playwright' to regenerate it.`,
+      );
+    }
+    copyFileSync(source, join(to, asset));
+  }
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  copyPreviewVendor();
+}

apps/desktop/src/electron/ElectronWindow.ts

@@ -16,6 +16,13 @@ export class ElectronWindowCreateError extends Data.TaggedError("ElectronWindowC
 }
 
 export interface ElectronWindowShape {
+  /**
+   * The usable area of the primary display, menu bar and dock excluded.
+   *
+   * Here rather than read from `Electron.screen` at the call site so window
+   * sizing stays testable without a display attached.
+   */
+  readonly workAreaSize: Effect.Effect<{ width: number; height: number }>;
   readonly create: (
     options: Electron.BrowserWindowConstructorOptions,
   ) => Effect.Effect<Electron.BrowserWindow, ElectronWindowCreateError>;
@@ -65,6 +72,7 @@ const make = Effect.gen(function* () {
   );
 
   return ElectronWindow.of({
+    workAreaSize: Effect.sync(() => Electron.screen.getPrimaryDisplay().workAreaSize),
     create: (options) =>
       Effect.try({
         try: () => new Electron.BrowserWindow(options),

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -48,6 +48,32 @@ import {
   setTheme,
   showContextMenu,
 } from "./methods/window.ts";
+import {
+  previewAttach,
+  previewClick,
+  previewDrag,
+  previewDetach,
+  previewEvaluate,
+  previewLocalServers,
+  previewMove,
+  previewClearBrowsingData,
+  previewClearCache,
+  previewOpenDevTools,
+  previewScreenshot,
+  previewCancelPick,
+  previewPickElement,
+  previewPress,
+  previewAwaitDrawing,
+  previewSetAnnotationMode,
+  previewRevealElement,
+  previewScroll,
+  previewSetColorScheme,
+  previewSetViewport,
+  previewSnapshot,
+  previewStatus,
+  previewType,
+  previewWaitFor,
+} from "./methods/preview.ts";
 
 export const installDesktopIpcHandlers = Effect.gen(function* () {
   const ipc = yield* DesktopIpc.DesktopIpc;
@@ -72,6 +98,31 @@ export const installDesktopIpcHandlers = Effect.gen(function* () {
   yield* ipc.handle(issueSshWebSocketToken);
   yield* ipc.handle(resolveSshPasswordPrompt);
 
+  yield* ipc.handle(previewAttach);
+  yield* ipc.handle(previewDetach);
+  yield* ipc.handle(previewStatus);
+  yield* ipc.handle(previewEvaluate);
+  yield* ipc.handle(previewSnapshot);
+  yield* ipc.handle(previewClick);
+  yield* ipc.handle(previewMove);
+  yield* ipc.handle(previewDrag);
+  yield* ipc.handle(previewType);
+  yield* ipc.handle(previewPress);
+  yield* ipc.handle(previewScroll);
+  yield* ipc.handle(previewWaitFor);
+  yield* ipc.handle(previewLocalServers);
+  yield* ipc.handle(previewScreenshot);
+  yield* ipc.handle(previewOpenDevTools);
+  yield* ipc.handle(previewSetColorScheme);
+  yield* ipc.handle(previewPickElement);
+  yield* ipc.handle(previewSetAnnotationMode);
+  yield* ipc.handle(previewAwaitDrawing);
+  yield* ipc.handle(previewCancelPick);
+  yield* ipc.handle(previewRevealElement);
+  yield* ipc.handle(previewSetViewport);
+  yield* ipc.handle(previewClearBrowsingData);
+  yield* ipc.handle(previewClearCache);
+
   yield* ipc.handle(getServerExposureState);
   yield* ipc.handle(setServerExposureMode);
   yield* ipc.handle(setTailscaleServeEnabled);

apps/desktop/src/ipc/channels.ts

@@ -1,6 +1,30 @@
 export const PICK_FOLDER_CHANNEL = "desktop:pick-folder";
 export const CONFIRM_CHANNEL = "desktop:confirm";
 export const CAPTURE_SCREENSHOT_CHANNEL = "desktop:capture-screenshot";
+export const PREVIEW_ATTACH_CHANNEL = "desktop:preview-attach";
+export const PREVIEW_DETACH_CHANNEL = "desktop:preview-detach";
+export const PREVIEW_STATUS_CHANNEL = "desktop:preview-status";
+export const PREVIEW_EVALUATE_CHANNEL = "desktop:preview-evaluate";
+export const PREVIEW_SNAPSHOT_CHANNEL = "desktop:preview-snapshot";
+export const PREVIEW_CLICK_CHANNEL = "desktop:preview-click";
+export const PREVIEW_MOVE_CHANNEL = "desktop:preview-move";
+export const PREVIEW_DRAG_CHANNEL = "desktop:preview-drag";
+export const PREVIEW_TYPE_CHANNEL = "desktop:preview-type";
+export const PREVIEW_PRESS_CHANNEL = "desktop:preview-press";
+export const PREVIEW_SCROLL_CHANNEL = "desktop:preview-scroll";
+export const PREVIEW_WAIT_FOR_CHANNEL = "desktop:preview-wait-for";
+export const PREVIEW_LOCAL_SERVERS_CHANNEL = "desktop:preview-local-servers";
+export const PREVIEW_SCREENSHOT_CHANNEL = "desktop:preview-screenshot";
+export const PREVIEW_OPEN_DEVTOOLS_CHANNEL = "desktop:preview-open-devtools";
+export const PREVIEW_SET_COLOR_SCHEME_CHANNEL = "desktop:preview-set-color-scheme";
+export const PREVIEW_PICK_ELEMENT_CHANNEL = "desktop:preview-pick-element";
+export const PREVIEW_CANCEL_PICK_CHANNEL = "desktop:preview-cancel-pick";
+export const PREVIEW_SET_ANNOTATION_MODE_CHANNEL = "desktop:preview-set-annotation-mode";
+export const PREVIEW_AWAIT_DRAWING_CHANNEL = "desktop:preview-await-drawing";
+export const PREVIEW_REVEAL_ELEMENT_CHANNEL = "desktop:preview-reveal-element";
+export const PREVIEW_SET_VIEWPORT_CHANNEL = "desktop:preview-set-viewport";
+export const PREVIEW_CLEAR_BROWSING_DATA_CHANNEL = "desktop:preview-clear-browsing-data";
+export const PREVIEW_CLEAR_CACHE_CHANNEL = "desktop:preview-clear-cache";
 export const SET_THEME_CHANNEL = "desktop:set-theme";
 export const SET_TASKBAR_STATUS_CHANNEL = "desktop:set-taskbar-status";
 export const CONTEXT_MENU_CHANNEL = "desktop:context-menu";