From 421d1dd728ffa97b3652ca98310ad93528ee048d Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Tue, 19 May 2026 11:01:38 -0700 Subject: [PATCH] feat(mcp): add video-show-actions / video-hide-actions tools and CLI commands Expose screencast `showActions`/`hideActions` through MCP (`browser_video_show_actions`, `browser_video_hide_actions`) and the CLI (`video-show-actions` with `--duration`/`--position`, `video-hide-actions`) so MCP/CLI-driven actions can be annotated on top of a recorded video. --- .../src/tools/backend/video.ts | 46 +++++++++++++++++++ .../src/tools/cli-client/skill/SKILL.md | 4 ++ .../src/tools/cli-daemon/commands.ts | 25 ++++++++++ tests/mcp/cli-devtools.spec.ts | 15 ++++++ tests/mcp/video.spec.ts | 43 +++++++++++++++++ 5 files changed, 133 insertions(+) diff --git a/packages/playwright-core/src/tools/backend/video.ts b/packages/playwright-core/src/tools/backend/video.ts index 7dcfe2508899f..5ed2237ae3905 100644 --- a/packages/playwright-core/src/tools/backend/video.ts +++ b/packages/playwright-core/src/tools/backend/video.ts @@ -94,8 +94,54 @@ const videoChapter = defineTool({ }, }); +const actionPosition = z.enum(['top-left', 'top', 'top-right', 'bottom-left', 'bottom', 'bottom-right']); + +const videoShowActions = defineTool({ + capability: 'devtools', + + schema: { + name: 'browser_video_show_actions', + title: 'Show action overlays', + description: 'Annotate subsequent actions performed on the page with a callout that names the action and highlights the target element. Useful while video recording or screencasting.', + inputSchema: z.object({ + duration: z.number().optional().describe('How long each action annotation stays on screen, in milliseconds. Defaults to 500.'), + position: actionPosition.optional().describe('Where to place the action title relative to the page. Defaults to top-right.'), + }), + type: 'readOnly', + }, + + handle: async (context, params, response) => { + const tab = context.currentTabOrDie(); + await tab.page.screencast.showActions({ + duration: params.duration, + position: params.position, + }); + response.addTextResult('Action annotations enabled.'); + }, +}); + +const videoHideActions = defineTool({ + capability: 'devtools', + + schema: { + name: 'browser_video_hide_actions', + title: 'Hide action overlays', + description: 'Stop annotating actions performed on the page.', + inputSchema: z.object({}), + type: 'readOnly', + }, + + handle: async (context, params, response) => { + const tab = context.currentTabOrDie(); + await tab.page.screencast.hideActions(); + response.addTextResult('Action annotations disabled.'); + }, +}); + export default [ videoStart, videoStop, videoChapter, + videoShowActions, + videoHideActions, ]; diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md index 6bc1cef82caa2..252e62ce4f294 100644 --- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md +++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md @@ -163,6 +163,10 @@ playwright-cli video-start video.webm playwright-cli video-chapter "Chapter Title" --description="Details" --duration=2000 playwright-cli video-stop +# annotate each subsequent action (click, type, ...) with a callout naming the action and highlighting the target +playwright-cli video-show-actions --duration=600 --position=top-right +playwright-cli video-hide-actions + # launch the dashboard for UI review / design feedback — user annotates the page, you receive the annotated screenshot, snapshot, and notes playwright-cli show --annotate diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts index a69b00ec0dc16..a280825b201f5 100644 --- a/packages/playwright-core/src/tools/cli-daemon/commands.ts +++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts @@ -972,6 +972,29 @@ const videoChapter = declareCommand({ toolParams: ({ title, description, duration }) => ({ title, description, duration }), }); +const actionPositionArg = z.enum(['top-left', 'top', 'top-right', 'bottom-left', 'bottom', 'bottom-right']); + +const videoShowActions = declareCommand({ + name: 'video-show-actions', + description: 'Annotate subsequent CLI/MCP actions on the page with a callout that names the action and highlights the target element', + category: 'devtools', + args: z.object({}), + options: z.object({ + duration: numberArg.optional().describe('How long each action annotation stays on screen, in milliseconds. Defaults to 500.'), + position: actionPositionArg.optional().describe('Where to place the action title: top-left, top, top-right, bottom-left, bottom, bottom-right. Defaults to top-right.'), + }), + toolName: 'browser_video_show_actions', + toolParams: ({ duration, position }) => ({ duration, position }), +}); + +const videoHideActions = declareCommand({ + name: 'video-hide-actions', + description: 'Stop annotating actions performed on the page', + category: 'devtools', + toolName: 'browser_video_hide_actions', + toolParams: () => ({}), +}); + const dashboardShow = declareCommand({ name: 'show', description: 'Show Playwright Dashboard', @@ -1201,6 +1224,8 @@ const commandsArray: AnyCommandSchema[] = [ videoStart, videoStop, videoChapter, + videoShowActions, + videoHideActions, dashboardShow, pauseAt, resume, diff --git a/tests/mcp/cli-devtools.spec.ts b/tests/mcp/cli-devtools.spec.ts index 7dabb7e0545cc..7718e6e714af9 100644 --- a/tests/mcp/cli-devtools.spec.ts +++ b/tests/mcp/cli-devtools.spec.ts @@ -236,6 +236,21 @@ test('video-chapter', async ({ cli, server }) => { await cli('video-stop'); }); +test('video-show-actions and video-hide-actions', async ({ cli, server }) => { + await cli('open', server.HELLO_WORLD); + const { output: showOutput } = await cli('video-show-actions', '--duration=200', '--position=bottom-right'); + expect(showOutput).toContain('Action annotations enabled.'); + const { output: hideOutput } = await cli('video-hide-actions'); + expect(hideOutput).toContain('Action annotations disabled.'); +}); + +test('video-show-actions rejects invalid position', async ({ cli, server }) => { + await cli('open', server.HELLO_WORLD); + const { error, exitCode } = await cli('video-show-actions', '--position=middle'); + expect(exitCode).not.toBe(0); + expect(error).toContain('position'); +}); + test('generate-locator', async ({ cli, server }) => { server.setContent('/', ``, 'text/html'); await cli('open', server.PREFIX); diff --git a/tests/mcp/video.spec.ts b/tests/mcp/video.spec.ts index 79043bdb5f0e5..3a1a0bbb67b32 100644 --- a/tests/mcp/video.spec.ts +++ b/tests/mcp/video.spec.ts @@ -95,6 +95,49 @@ test('reports missing ffmpeg, not missing browser, when recordVideo is enabled', }); }); +test.describe('action overlays', () => { + test.use({ mcpArgs: ['--caps=devtools'] }); + + test('browser_video_show_actions and browser_video_hide_actions', async ({ client, server }) => { + expect(await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.HELLO_WORLD }, + })).toHaveResponse({ + code: expect.stringContaining(`page.goto('http://localhost`), + }); + + expect(await client.callTool({ + name: 'browser_video_show_actions', + arguments: { duration: 200, position: 'bottom-right' }, + })).toHaveResponse({ + result: 'Action annotations enabled.', + }); + + expect(await client.callTool({ + name: 'browser_video_hide_actions', + arguments: {}, + })).toHaveResponse({ + result: 'Action annotations disabled.', + }); + }); + + test('browser_video_show_actions rejects invalid position', async ({ client, server }) => { + expect(await client.callTool({ + name: 'browser_navigate', + arguments: { url: server.HELLO_WORLD }, + })).toHaveResponse({ + code: expect.stringContaining(`page.goto('http://localhost`), + }); + + expect(await client.callTool({ + name: 'browser_video_show_actions', + arguments: { position: 'middle' }, + })).toHaveResponse({ + isError: true, + }); + }); +}); + async function produceFrames(client: Client) { expect(await client.callTool({ name: 'browser_evaluate',