diff --git a/README.md b/README.md index bccbe30..b7c3f49 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Shadow DOM + iframes), multi-step plans, structured extraction, visual diff, and guardrails** for payments and bookings. It drives real Chromium, so it reads **Next.js / SPA** pages after hydration — not just static HTML. -> 30 MCP tools · stealth + rotating proxies · virtualized-list scraping · HAR record/replay · pixel visual-diff · human handoff. +> 32 MCP tools · stealth + rotating proxies · virtualized-list scraping · HAR record/replay · pixel visual-diff · human handoff + live view. ## Install @@ -55,7 +55,7 @@ agent passes `humanApproved`. - **Sees everything** — open Shadow DOM, same/cross-origin iframes, and **virtualized/infinite lists** (`browser_collect`). - **Fast-path** — `browser_fetch` impersonates a real Chrome TLS fingerprint for server-rendered HTML, no browser launch — returns clean **markdown** and optional **contacts** (`extractContacts`) at ~HTTP speed. - **Data out** — multi-currency prices, typed CSS extraction, **contact extraction** (emails/phones E.164, `fastPathFirst` cascade), a clean→validate→dedupe→emit pipeline, CSV export, Google SERP rank tracking. -- **Ops** — persistent sessions, **auto crash recovery** (a crashed page is recreated in the same context and restored to its last URL between calls), opt-in **per-host circuit breaker** + **bounded probe queue/budget** for mass scraping, `storageState` auto-save, HAR record/replay, pixel `visual_diff`, human handoff for login/2FA. +- **Ops** — persistent sessions, **auto crash recovery** (a crashed page is recreated in the same context and restored to its last URL between calls), opt-in **per-host circuit breaker** + **bounded probe queue/budget** + **`browser_metrics`** for mass scraping, **live view** (watch any session — even headless — in your browser), `storageState` auto-save, HAR record/replay, pixel `visual_diff`, human handoff for login/2FA. ## Documentation @@ -63,7 +63,7 @@ Full reference in **[`docs/`](./docs/README.md)**: [Installation](./docs/installation.md) · [CLI](./docs/cli.md) · -[MCP tools (30)](./docs/mcp-tools.md) · +[MCP tools (32)](./docs/mcp-tools.md) · [Configuration](./docs/configuration.md) · [Sessions](./docs/sessions.md) · [Extraction](./docs/extraction.md) · diff --git a/docs/README.md b/docs/README.md index bf87b48..3d616d0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,7 +7,7 @@ New here? Start with the root [README](../README.md), then dive in: | --- | --- | | [Installation](./installation.md) | Requirements, install, Chromium, MCP registration, the three ways to get a browser | | [CLI](./cli.md) | `probe` / `fetch` / `serp-batch` / `shots` + every flag | -| [MCP tools](./mcp-tools.md) | All 30 tools with parameters and examples | +| [MCP tools](./mcp-tools.md) | All 32 tools with parameters and examples | | [Configuration](./configuration.md) | `AgentOptions`, `FUSE_*` env vars, identity, retry, output location | | [Sessions](./sessions.md) | Session lifecycle, auto crash recovery, `storageState` auto-save, HAR record/replay, CDP attach | | [Extraction](./extraction.md) | `browser_extract` / `extract_schema` / `collect` + the clean→validate→dedupe→emit pipeline | diff --git a/docs/mcp-tools.md b/docs/mcp-tools.md index 611b6ce..3ca67f0 100644 --- a/docs/mcp-tools.md +++ b/docs/mcp-tools.md @@ -1,6 +1,6 @@ # MCP tools -Complete reference for the 30 `browser_*` tools exposed by the fuse-browser MCP server. +Complete reference for the 32 `browser_*` tools exposed by the fuse-browser MCP server. Tools fall into two families: @@ -560,3 +560,35 @@ Returned fields: `uptimeMs`, `probesOk`, `probesFailed`, `avgDurationMs`, `minDu ```json { "reset": false } ``` + +--- + +## Live view + +Watch a session's browser in real time from a normal web page — works even for **headless** sessions. A CDP screencast streams JPEG frames over an ephemeral, token-gated `127.0.0.1` server (Server-Sent Events) to a ``/`` viewer. **Read-only** (no click-through). The frame stream binds to the page at start; after a crash recovery it does not auto-reattach — call `browser_live_view` again. + +### browser_live_view + +Start the live view for a session and return the viewer URL (token embedded). Opens it in the OS default browser unless `open:false`. + +| Param | Type | Required | Description | +| --- | --- | --- | --- | +| `sessionId` | string | yes | Target session. | +| `quality` | integer | no | JPEG quality 1–100 (default `60`). | +| `maxWidth` | integer | no | Max frame width in px (default `1280`). | +| `maxHeight` | integer | no | Max frame height in px (default `720`). | +| `open` | boolean | no | Open the URL in the default browser (default `true`). | + +Returns `{ url, note }`. The server auto-closes when the page closes or on `browser_live_view_stop`. + +```json +{ "sessionId": "s_abc123", "quality": 60, "open": true } +``` + +### browser_live_view_stop + +Stop a session's live view and shut down its local server. Returns `{ stopped: true|false }` (`false` if none was running). + +```json +{ "sessionId": "s_abc123" } +``` diff --git a/package.json b/package.json index 96aa39f..73844fb 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", "test": "bun test tests/unit", - "test:integration": "node --test --import tsx tests/integration/mcp.test.ts tests/integration/probe.test.ts tests/integration/snapshot.test.ts tests/integration/snapshot-frames.test.ts tests/integration/collect.test.ts tests/integration/selectors.test.ts tests/integration/visual-diff.test.ts tests/integration/session-state.test.ts tests/integration/pipeline.test.ts tests/integration/run.test.ts tests/integration/extract-schema.test.ts tests/integration/recovery.test.ts", + "test:integration": "node --test --import tsx tests/integration/mcp.test.ts tests/integration/probe.test.ts tests/integration/snapshot.test.ts tests/integration/snapshot-frames.test.ts tests/integration/collect.test.ts tests/integration/selectors.test.ts tests/integration/visual-diff.test.ts tests/integration/session-state.test.ts tests/integration/pipeline.test.ts tests/integration/run.test.ts tests/integration/extract-schema.test.ts tests/integration/recovery.test.ts tests/integration/live-view.test.ts", "browsers": "patchright install chromium", "mcp": "node --import tsx src/bin/mcp.ts", "cli": "node --import tsx src/bin/cli.ts" diff --git a/src/live/manager.ts b/src/live/manager.ts new file mode 100644 index 0000000..538abb3 --- /dev/null +++ b/src/live/manager.ts @@ -0,0 +1,60 @@ +/** + * Live-view manager: ties a session to an ephemeral SSE server fed by a CDP + * screencast of its page. One live view per session; starting again replaces + * the previous one. Auto-stops when the page closes. + * @module live/manager + */ +import { randomBytes } from "node:crypto"; +import type { SessionData } from "../session/session.js"; +import { openUrl } from "./open-url.js"; +import { startScreencast, type ScreencastOptions } from "./screencast.js"; +import { type LiveServer, startSseServer } from "./sse-server.js"; + +interface LiveView { + url: string; + stop: () => Promise; +} + +const views = new Map(); + +/** Options for {@link startLiveView}. */ +export interface LiveViewOptions extends ScreencastOptions { + /** Open the viewer in the OS default browser (default true). */ + open: boolean; +} + +/** + * Start (or restart) the live view for a session and return the viewer URL. + * + * @param session - The live session whose page to stream. + * @param opts - Frame quality/size and whether to auto-open the viewer. + * @remarks The screencast binds to the page live at call time. If the page is + * later recreated by crash recovery (B1), the stream tears down with the old + * page (no silent leak) but does not auto-reattach — call `browser_live_view` + * again after a recovery to resume watching. + */ +export async function startLiveView(session: SessionData, opts: LiveViewOptions): Promise { + await stopLiveView(session.id); + const token = randomBytes(16).toString("hex"); + const server: LiveServer = await startSseServer(token); + const stopCast = await startScreencast(session.page, opts, (b64) => server.broadcast(b64)); + views.set(session.id, { + url: server.url, + stop: async () => { + await stopCast(); + await server.close(); + }, + }); + session.page.once("close", () => void stopLiveView(session.id)); + if (opts.open) openUrl(server.url); + return server.url; +} + +/** Stop a session's live view; returns false if none was running. */ +export async function stopLiveView(id: string): Promise { + const view = views.get(id); + if (!view) return false; + views.delete(id); + await view.stop(); + return true; +} diff --git a/src/live/open-url.ts b/src/live/open-url.ts new file mode 100644 index 0000000..7c81098 --- /dev/null +++ b/src/live/open-url.ts @@ -0,0 +1,17 @@ +/** + * Best-effort: open a URL in the OS default browser. Never throws. + * @module live/open-url + */ +import { spawn } from "node:child_process"; + +/** Spawn the platform opener for `url`, detached; failures are ignored. */ +export function openUrl(url: string): void { + const platform = process.platform; + const cmd = platform === "darwin" ? "open" : platform === "win32" ? "cmd" : "xdg-open"; + const args = platform === "win32" ? ["/c", "start", "", url] : [url]; + try { + spawn(cmd, args, { detached: true, stdio: "ignore" }).unref(); + } catch { + /* best-effort: the URL is also returned to the caller */ + } +} diff --git a/src/live/screencast.ts b/src/live/screencast.ts new file mode 100644 index 0000000..4e8733b --- /dev/null +++ b/src/live/screencast.ts @@ -0,0 +1,59 @@ +/** + * CDP screencast on a live page: emits base64 JPEG frames to a callback. Each + * frame is acked with its own `sessionId` (mandatory — the stream stalls after + * frame 1 otherwise), and the screencast is re-issued on navigation (it stops + * on a hard nav). Chromium only; works headless. + * @module live/screencast + */ +import type { CDPSession, Page } from "playwright"; + +/** Screencast frame quality/size knobs. */ +export interface ScreencastOptions { + quality: number; + maxWidth: number; + maxHeight: number; +} + +interface ScreencastFrame { + data: string; + sessionId: number; +} + +/** + * Start a CDP screencast on `page`, invoking `onFrame` with each base64 JPEG. + * + * @param page - The live Playwright page (Chromium). + * @param opts - Frame quality and max dimensions. + * @param onFrame - Called with the base64 JPEG of every frame. + * @returns A stop function that ends the screencast and detaches the session. + */ +export async function startScreencast( + page: Page, + opts: ScreencastOptions, + onFrame: (base64: string) => void, +): Promise<() => Promise> { + const cdp: CDPSession = await page.context().newCDPSession(page); + const begin = (): Promise => + cdp + .send("Page.startScreencast", { + format: "jpeg", + quality: opts.quality, + maxWidth: opts.maxWidth, + maxHeight: opts.maxHeight, + everyNthFrame: 1, + }) + .catch(() => undefined); + cdp.on("Page.screencastFrame", (frame: ScreencastFrame) => { + // Ack FIRST (with the frame's own sessionId) or the stream stalls. + void cdp.send("Page.screencastFrameAck", { sessionId: frame.sessionId }).catch(() => undefined); + onFrame(frame.data); + }); + const onLoad = (): void => void begin(); + page.on("load", onLoad); + await begin(); + return async () => { + page.off("load", onLoad); + await cdp.send("Page.stopScreencast").catch(() => undefined); + await cdp.detach().catch(() => undefined); + }; +} diff --git a/src/live/sse-server.ts b/src/live/sse-server.ts new file mode 100644 index 0000000..464ece9 --- /dev/null +++ b/src/live/sse-server.ts @@ -0,0 +1,66 @@ +/** + * Ephemeral localhost SSE server for the live view. Binds 127.0.0.1 on an + * OS-assigned port, gates every request on a token, serves the viewer HTML at + * `/` and a `text/event-stream` of base64 JPEG frames at `/stream`. + * @module live/sse-server + */ +import { createServer, type ServerResponse } from "node:http"; +import { VIEWER_HTML } from "./viewer-html.js"; + +/** A running live-view server. */ +export interface LiveServer { + /** Viewer URL (includes the access token). */ + url: string; + /** Push a base64 JPEG frame to every connected viewer. */ + broadcast(base64: string): void; + /** Close all streams and stop the server. */ + close(): Promise; +} + +/** Start the token-gated SSE server bound to loopback. */ +export async function startSseServer(token: string): Promise { + const clients = new Set(); + const server = createServer((req, res) => { + const u = new URL(req.url ?? "/", "http://127.0.0.1"); + if (u.searchParams.get("token") !== token) { + res.writeHead(403).end("forbidden"); + return; + } + if (u.pathname === "/stream") { + res.writeHead(200, { + "content-type": "text/event-stream", + "cache-control": "no-cache", + connection: "keep-alive", + }); + clients.add(res); + req.on("close", () => clients.delete(res)); + return; + } + res.writeHead(200, { "content-type": "text/html; charset=utf-8" }).end(VIEWER_HTML); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + return { + url: `http://127.0.0.1:${port}/?token=${token}`, + broadcast(base64) { + const chunk = `data: ${base64}\n\n`; + for (const res of clients) { + if (res.writableEnded || res.destroyed) { + clients.delete(res); + continue; + } + // Async EPIPE/ECONNRESET (client gone mid-frame) arrives in the callback, + // not as a throw — handle it here so it never crashes the process. + res.write(chunk, (err) => { + if (err) clients.delete(res); + }); + } + }, + async close() { + for (const res of clients) res.end(); + clients.clear(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} diff --git a/src/live/viewer-html.ts b/src/live/viewer-html.ts new file mode 100644 index 0000000..bb5ca03 --- /dev/null +++ b/src/live/viewer-html.ts @@ -0,0 +1,35 @@ +/** + * Self-contained HTML viewer for the live screencast. Connects to the SSE + * `/stream` endpoint (token from the URL) and paints each base64 JPEG frame + * into a full-window . Read-only, no dependencies. + * @module live/viewer-html + */ + +/** The viewer page served at `/` (inlined; no file reads). */ +export const VIEWER_HTML = ` + + + + +fuse-browser · live view + + + +
connecting…
+
live view
+ + +`; diff --git a/src/server/server.ts b/src/server/server.ts index 05c2ba9..939440e 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -13,6 +13,7 @@ import { registerExtractTool } from "./tools/extract.js"; import { registerExtractSchemaTool } from "./tools/extract-schema.js"; import { registerHandoffTool } from "./tools/handoff.js"; import { registerInspectTool } from "./tools/inspect.js"; +import { registerLiveViewTool } from "./tools/live-view.js"; import { registerMetricsTool } from "./tools/metrics.js"; import { registerNavigateTool } from "./tools/navigate.js"; import { registerFetchTool } from "./tools/fetch.js"; @@ -52,6 +53,7 @@ export function createServer(): BuiltServer { registerInspectTool(server, sessions); registerVisualDiffTool(server, sessions); registerHandoffTool(server, sessions); + registerLiveViewTool(server, sessions); registerMetricsTool(server); registerResources(server); return { server, sessions }; diff --git a/src/server/tools/live-view.ts b/src/server/tools/live-view.ts new file mode 100644 index 0000000..ae7fbff --- /dev/null +++ b/src/server/tools/live-view.ts @@ -0,0 +1,56 @@ +/** + * `browser_live_view` / `browser_live_view_stop`: stream a session's browser to + * a local web page so a human can watch in real time (read-only). Starts an + * ephemeral 127.0.0.1 server fed by a CDP screencast; the URL carries an access + * token. Works for headless sessions too. + * @module server/tools/live-view + */ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { startLiveView, stopLiveView } from "../../live/manager.js"; +import type { SessionManager } from "../../session/manager.js"; +import { jsonResult } from "../result.js"; +import { withSession } from "./with-session.js"; + +/** Register the live-view tools. */ +export function registerLiveViewTool(server: McpServer, sessions: SessionManager): void { + server.registerTool( + "browser_live_view", + { + title: "Live view (human)", + description: + "Stream a session's browser to a local web page (read-only) so a human can watch in real time — works headless too. Returns a 127.0.0.1 URL with an access token and opens it in the default browser (open:false to skip). Stop with browser_live_view_stop.", + inputSchema: { + sessionId: z.string(), + quality: z.number().int().min(1).max(100).optional(), + maxWidth: z.number().int().optional(), + maxHeight: z.number().int().optional(), + open: z.boolean().optional(), + }, + }, + async (a) => + withSession(sessions, String(a.sessionId), async (s) => { + const url = await startLiveView(s, { + quality: typeof a.quality === "number" ? a.quality : 60, + maxWidth: typeof a.maxWidth === "number" ? a.maxWidth : 1280, + maxHeight: typeof a.maxHeight === "number" ? a.maxHeight : 720, + open: a.open !== false, + }); + return jsonResult({ url, note: "Read-only live view. Open the URL to watch; the token is embedded." }); + }), + ); + + server.registerTool( + "browser_live_view_stop", + { + title: "Stop live view", + description: "Stop the live view for a session and shut down its local server.", + inputSchema: { sessionId: z.string() }, + }, + async (a) => + withSession(sessions, String(a.sessionId), async (s) => { + const stopped = await stopLiveView(s.id); + return jsonResult({ stopped }); + }), + ); +} diff --git a/tests/integration/live-view.test.ts b/tests/integration/live-view.test.ts new file mode 100644 index 0000000..c29849b --- /dev/null +++ b/tests/integration/live-view.test.ts @@ -0,0 +1,60 @@ +/** + * End-to-end test for the human live view: start a CDP screencast on a real + * headless session, connect to the ephemeral SSE server, and assert at least + * one base64 JPEG frame arrives. Then stop and verify teardown. + */ +import assert from "node:assert/strict"; +import { get } from "node:http"; +import { test } from "node:test"; +import { resolveConfig } from "../../src/agent/config.js"; +import { startLiveView, stopLiveView } from "../../src/live/manager.js"; +import { SessionManager } from "../../src/session/manager.js"; + +/** Resolve with the first SSE frame's base64 payload, or reject on timeout. */ +function firstFrame(streamUrl: string): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + req.destroy(); + reject(new Error("no frame within 15s")); + }, 15_000); + const req = get(streamUrl, (res) => { + let buf = ""; + res.on("data", (chunk: Buffer) => { + buf += chunk.toString("utf8"); + const m = buf.match(/data: (.+)\n\n/); + if (m?.[1]) { + clearTimeout(timer); + req.destroy(); + resolve(m[1]); + } + }); + }); + req.on("error", (err) => { + clearTimeout(timer); + reject(err); + }); + }); +} + +test("live view streams screencast frames over SSE then stops", { timeout: 120_000 }, async () => { + const sessions = new SessionManager(); + const s = await sessions.open(resolveConfig({ headless: true, engine: "patchright" })); + try { + await s.page.goto("data:text/html,

LIVE

", { + waitUntil: "domcontentloaded", + timeout: 30_000, + }); + const url = await startLiveView(s, { quality: 50, maxWidth: 640, maxHeight: 480, open: false }); + assert.match(url, /^http:\/\/127\.0\.0\.1:\d+\/\?token=[a-f0-9]{32}$/, "viewer URL shape"); + + const streamUrl = url.replace("/?token=", "/stream?token="); + const frame = await firstFrame(streamUrl); + assert.ok(frame.length > 100, "received a non-trivial base64 JPEG frame"); + assert.doesNotThrow(() => Buffer.from(frame, "base64"), "frame is valid base64"); + + assert.equal(await stopLiveView(s.id), true, "stop returns true when running"); + assert.equal(await stopLiveView(s.id), false, "stop is idempotent"); + } finally { + await sessions.close(s.id); + } +}); diff --git a/tests/integration/mcp.test.ts b/tests/integration/mcp.test.ts index f88616a..bb95c4f 100644 --- a/tests/integration/mcp.test.ts +++ b/tests/integration/mcp.test.ts @@ -49,6 +49,8 @@ const EXPECTED = [ "browser_handoff", "browser_serp_batch", "browser_metrics", + "browser_live_view", + "browser_live_view_stop", ]; test("MCP exposes the expected tool set with no duplicates", async () => {