From 9b9eedb506bc1b5bce9e9a3d6b6803bac69d8242 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Thu, 16 Apr 2026 21:38:54 +0800
Subject: [PATCH 01/12] Add image input for user prompts and upload_file
 browser action
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Users can now attach images to a task (paste, drag-drop, or paperclip)
and the agent can attach local files to <input type=file> controls via
CDP DOM.setFileInputFiles, bypassing the native OS picker.

- Frontend: paste/drop/picker → data URIs in POST body, up to 8 images
  at 10MB each; thumbnails with × remove.
- Server: validates data URIs and size, builds multimodal Message
  (TextContent + ImageContent) in both in-process and multi-process
  dispatch paths.
- Extension: new `uploadable` element type with a dedicated detection
  pass that surfaces display:none file inputs and anchors the overlay
  on the nearest visible label/button. New `upload_file` action on
  ElementInteractionTool resolves selector → CDP nodeId and calls
  DOM.setFileInputFiles.
- Prompts + highlight tool updated for big-model and small-model to
  advertise the new element type and action.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 extension/src/background/index.ts             |  40 ++-
 extension/src/commands/element-actions.ts     | 114 +++++++
 .../commands/highlight-detection.injected.js  | 181 ++++++++++-
 extension/src/types.ts                        |  16 +
 frontend/index.html                           | 290 +++++++++++++++++-
 server/agent/api.py                           |  64 +++-
 .../big_model/element_interaction_tool.j2     |  18 +-
 .../agent/prompts/big_model/highlight_tool.j2 |   4 +-
 .../small_model/element_interaction_tool.j2   |   9 +
 .../prompts/small_model/highlight_tool.j2     |   4 +-
 server/agent/tools/browser_executor.py        |  21 ++
 .../agent/tools/element_interaction_tool.py   |  13 +-
 server/agent/tools/highlight_tool.py          |   2 +-
 server/api/routes/agent.py                    |  84 ++++-
 server/core/browser_executor_bundle.py        |  18 +-
 server/core/process_manager.py                |   1 +
 server/core/processor.py                      |  36 +++
 server/models/commands.py                     |  26 +-
 .../tests/unit/test_agent_api_multiprocess.py |   1 +
 server/tests/unit/test_api_uuid.py            |   5 +-
 20 files changed, 916 insertions(+), 31 deletions(-)

diff --git a/extension/src/background/index.ts b/extension/src/background/index.ts
index 155bc87..b35132c 100644
--- a/extension/src/background/index.ts
+++ b/extension/src/background/index.ts
@@ -37,6 +37,7 @@ import {
   performElementSwipe,
   performElementDragAndDrop,
   performElementSetSlider,
+  performElementUpload,
   performKeyboardInput,
   performElementSelect,
   replayHoverState,
@@ -295,6 +296,7 @@ const IN_PAGE_HIGHLIGHT_COLORS: Record<string, { border: string; bg: string }> =
     selectable: { border: '#FF6B6B', bg: 'rgba(255,107,107,0.7)' },
     draggable: { border: '#FF6600', bg: 'rgba(255,102,0,0.7)' },
     droppable: { border: '#339966', bg: 'rgba(51,153,102,0.7)' },
+    uploadable: { border: '#AA66FF', bg: 'rgba(170,102,255,0.7)' },
     any: { border: '#00CCCC', bg: 'rgba(0,204,204,0.7)' },
   };
 
@@ -307,7 +309,10 @@ function buildInPageHighlightScript(elements: InteractiveElement[]): string {
       IN_PAGE_HIGHLIGHT_COLORS[el.type] || IN_PAGE_HIGHLIGHT_COLORS.clickable;
     return {
       id: el.id,
-      selector: el.selector,
+      // The overlay script renders the box on `selector`. For uploadable
+      // file inputs that are display:none, the visible anchor's selector
+      // lets the overlay land on something the user can actually see.
+      selector: el.overlaySelector || el.selector,
       borderColor: colors.border,
       bgColor: colors.bg,
       labelPos: el.labelPosition || 'above',
@@ -953,6 +958,7 @@ function isHeavyBrowserCommand(data: any): boolean {
     case 'set_slider_value':
     case 'keyboard_input':
     case 'select_element':
+    case 'upload_file':
     case 'handle_dialog':
       return true;
     case 'tab':
@@ -2328,6 +2334,38 @@ async function handleCommand(command: Command): Promise<CommandResponse> {
         };
       }
 
+      case 'upload_file': {
+        if (!command.conversation_id)
+          throw new Error('conversation_id required');
+        const uploadTabId = command.tab_id;
+        if (uploadTabId === undefined || uploadTabId === null)
+          throw new Error('tab_id is required');
+        if (!command.file_path || typeof command.file_path !== 'string')
+          throw new Error('file_path is required for upload_file');
+
+        const uploadResult = await performElementUpload(
+          command.conversation_id,
+          command.element_id,
+          uploadTabId,
+          command.file_path,
+        );
+        const uploadPageState = await captureDefaultHighlightedPageState({
+          tabId: uploadTabId,
+          conversationId: command.conversation_id,
+          logLabel: 'UploadFile',
+        });
+
+        return {
+          success: uploadResult.success,
+          data: {
+            ...uploadResult,
+            ...uploadPageState,
+          },
+          error: uploadResult.error,
+          timestamp: Date.now(),
+        };
+      }
+
       case 'keyboard_input': {
         if (!command.conversation_id)
           throw new Error('conversation_id required');
diff --git a/extension/src/commands/element-actions.ts b/extension/src/commands/element-actions.ts
index e8a1797..1561422 100644
--- a/extension/src/commands/element-actions.ts
+++ b/extension/src/commands/element-actions.ts
@@ -11,6 +11,7 @@ import type { ElementActionResult } from '../types';
  * - Handles dialog events using the same pattern as javascript.ts
  */
 
+import { CdpCommander } from './cdp-commander';
 import { buildElementCacheMissMessage, elementCache } from './element-cache';
 import { executeJavaScript, type JavaScriptResult } from './javascript';
 import { buildHitTestVisibilityHelpersScript } from '../utils/hit-test-visibility';
@@ -516,6 +517,15 @@ export interface HoverResult extends ElementActionResult {
   error?: string;
 }
 
+/**
+ * Result type for file upload operation
+ */
+export interface UploadResult extends ElementActionResult {
+  uploaded: boolean;
+  staleElement?: boolean;
+  error?: string;
+}
+
 /**
  * Result type for element select operation
  */
@@ -4096,6 +4106,109 @@ export async function performElementSelect(
   return result;
 }
 
+/**
+ * Attach a local file (by absolute path on the host) to an <input type="file">
+ * via CDP `DOM.setFileInputFiles`. This bypasses the native OS file picker —
+ * attempting to click the input would pop the picker in front of the user,
+ * which the agent cannot drive.
+ *
+ * The server validates the path before dispatching, so here we only need to
+ * resolve the cached selector to a CDP `nodeId` and invoke setFileInputFiles.
+ */
+export async function performElementUpload(
+  conversationId: string,
+  elementId: string,
+  tabId: number,
+  filePath: string,
+): Promise<UploadResult> {
+  console.log(
+    `📎 [ElementUpload] Uploading "${filePath}" to element ${elementId} on tab ${tabId}`,
+  );
+
+  const cachedElement = elementCache.getElementById(
+    conversationId,
+    tabId,
+    elementId,
+  );
+  if (!cachedElement) {
+    console.log(`❌ [ElementUpload] Element ${elementId} not found in cache`);
+    return {
+      success: false,
+      ...buildResolvedElementResultFields(elementId, elementId),
+      uploaded: false,
+      staleElement: false,
+      error: buildElementCacheMissMessage({
+        conversationId,
+        tabId,
+        elementId,
+      }),
+    };
+  }
+
+  const element = cachedElement.element;
+  const resolvedElementFields = buildResolvedElementResultFields(
+    cachedElement.requestedElementId,
+    cachedElement.resolvedElementId,
+  );
+  const cdp = new CdpCommander(tabId);
+
+  try {
+    // Resolve selector → CDP nodeId. DOM.getDocument returns the document root
+    // node; DOM.querySelector is scoped to that root and accepts any CSS
+    // selector. A nodeId of 0 indicates no match (selector went stale).
+    const doc = (await cdp.sendCommand('DOM.getDocument', { depth: 0 })) as {
+      root?: { nodeId: number };
+    };
+    if (!doc || !doc.root || typeof doc.root.nodeId !== 'number') {
+      return {
+        success: false,
+        ...resolvedElementFields,
+        uploaded: false,
+        error: 'CDP DOM.getDocument returned no root node',
+      };
+    }
+
+    const queryResult = (await cdp.sendCommand('DOM.querySelector', {
+      nodeId: doc.root.nodeId,
+      selector: element.selector,
+    })) as { nodeId?: number };
+
+    if (!queryResult || !queryResult.nodeId) {
+      return {
+        success: false,
+        ...resolvedElementFields,
+        uploaded: false,
+        staleElement: true,
+        error: `Selector "${element.selector}" no longer resolves to a DOM node (element became stale).`,
+      };
+    }
+
+    await cdp.sendCommand('DOM.setFileInputFiles', {
+      nodeId: queryResult.nodeId,
+      files: [filePath],
+    });
+
+    console.log(
+      `✅ [ElementUpload] DOM.setFileInputFiles succeeded for ${elementId} (${filePath})`,
+    );
+
+    return {
+      success: true,
+      ...resolvedElementFields,
+      uploaded: true,
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    console.error(`❌ [ElementUpload] failed: ${message}`);
+    return {
+      success: false,
+      ...resolvedElementFields,
+      uploaded: false,
+      error: message,
+    };
+  }
+}
+
 /**
  * Export element actions module
  */
@@ -4105,4 +4218,5 @@ export const elementActions = {
   performElementScroll,
   performKeyboardInput,
   performElementSelect,
+  performElementUpload,
 };
diff --git a/extension/src/commands/highlight-detection.injected.js b/extension/src/commands/highlight-detection.injected.js
index c800fff..ad31ef5 100644
--- a/extension/src/commands/highlight-detection.injected.js
+++ b/extension/src/commands/highlight-detection.injected.js
@@ -5,6 +5,7 @@ const HIGHLIGHT_TYPE_PRIORITY = {
   scrollable: 3,
   draggable: 4,
   droppable: 5,
+  uploadable: 6,
 };
 
 const HIGHLIGHT_SIGNAL_SCORE = {
@@ -16,6 +17,7 @@ const HIGHLIGHT_SIGNAL_SCORE = {
   inputable: 360,
   selectable: 340,
   scrollable: 220,
+  uploadable: 360,
 };
 
 const POINTER_ROLE_SET = new Set([
@@ -812,6 +814,69 @@ function isSelectableCandidate(el) {
   return !isDisabledForDetection(el) && el.tagName.toLowerCase() === 'select';
 }
 
+/**
+ * Match <input type="file"> regardless of visibility. File inputs are almost
+ * always hidden (display:none / size:0) behind a styled <label> or wrapping
+ * <button> that forwards clicks via the DOM. Upload is dispatched to the
+ * underlying input via CDP DOM.setFileInputFiles, so we do not need the
+ * element to be visible — only present.
+ */
+function isUploadableCandidate(el) {
+  if (isDisabledForDetection(el)) {
+    return false;
+  }
+  if (el.tagName.toLowerCase() !== 'input') {
+    return false;
+  }
+  const inputType = (el.getAttribute('type') || '').toLowerCase();
+  return inputType === 'file';
+}
+
+/**
+ * Find the best visible DOM anchor to draw a highlight overlay on for a
+ * (usually invisible) file input. Preference:
+ *   1. <label for="<id>"> explicitly associated with the input.
+ *   2. An ancestor <label> that contains this input.
+ *   3. The nearest visible wrapping element (button, clickable div, form).
+ * Returns null if nothing visible is found (the caller should skip the input).
+ */
+function findUploadVisualAnchor(fileInput) {
+  try {
+    if (fileInput.id) {
+      const explicitLabel = document.querySelector(
+        `label[for="${CSS.escape(fileInput.id)}"]`,
+      );
+      if (explicitLabel && isElementVisibleForDetection(explicitLabel)) {
+        return explicitLabel;
+      }
+    }
+  } catch (_error) {
+    // CSS.escape failure or invalid selector — fall through to ancestor walk.
+  }
+
+  const containingLabel = fileInput.closest('label');
+  if (containingLabel && isElementVisibleForDetection(containingLabel)) {
+    return containingLabel;
+  }
+
+  let ancestor = fileInput.parentElement;
+  let depth = 0;
+  while (ancestor && depth < 6) {
+    if (
+      ancestor instanceof HTMLElement &&
+      isElementVisibleForDetection(ancestor)
+    ) {
+      const rect = ancestor.getBoundingClientRect();
+      if (rect.width > 0 && rect.height > 0) {
+        return ancestor;
+      }
+    }
+    ancestor = ancestor.parentElement;
+    depth += 1;
+  }
+  return null;
+}
+
 function hasStructuredInteractiveDescendant(el) {
   const stack = [];
   for (const child of Array.from(el.children)) {
@@ -1745,6 +1810,13 @@ function resolveElementCandidate(el, requestedType) {
       : null;
   }
 
+  if (requestedType === 'uploadable') {
+    // Uploadable elements are handled out-of-band by collectHighlightCandidates
+    // because file inputs are typically display:none and would be filtered out
+    // before reaching this resolver.
+    return null;
+  }
+
   if (requestedType === 'any') {
     const candidates = [];
 
@@ -1959,8 +2031,15 @@ function toInteractiveElement(candidate) {
       ? 'scrollable'
       : candidate.type;
   const text = getElementTextForDetection(candidate.element);
-
-  return {
+  // For `uploadable`, the candidate element is usually a display:none file
+  // input whose own bbox is zero — use the visible anchor rect captured in
+  // collectUploadableCandidates instead.
+  const bbox =
+    candidate.type === 'uploadable' && candidate.rect
+      ? candidate.rect
+      : getElementRect(candidate.element);
+
+  const base = {
     id: '',
     type: displayType,
     ...(interactionHints.length > 0 ? { interactionHints } : {}),
@@ -1972,10 +2051,20 @@ function toInteractiveElement(candidate) {
     text,
     searchText: getElementSearchText(candidate.element),
     fingerprint: getElementFingerprint(candidate.element),
-    bbox: getElementRect(candidate.element),
+    bbox,
     isVisible: true,
     isInViewport: true,
   };
+
+  // Uploadable inputs are hidden; the overlay renderer queries by selector
+  // and skips when rect is 0×0. Expose the visible anchor's selector so the
+  // renderer can draw the box on the anchor while actions still target the
+  // underlying <input type="file">.
+  if (candidate.type === 'uploadable' && candidate.overlayElement) {
+    base.overlaySelector = generateSelector(candidate.overlayElement);
+  }
+
+  return base;
 }
 
 function countVisibleClickableCandidates(metricsStartTime) {
@@ -2338,9 +2427,94 @@ function getCandidateElementsForScan(layoutStability, trace, config) {
   return cappedElements;
 }
 
+function collectUploadableCandidates(trace) {
+  // File inputs are usually display:none; the standard scan filters them out.
+  // This pass queries them directly and anchors the overlay on the nearest
+  // visible ancestor (label/wrapping button) while keeping the underlying
+  // <input type="file"> as the action target for CDP DOM.setFileInputFiles.
+  const fileInputs = document.querySelectorAll('input[type="file"]');
+  const candidates = [];
+
+  fileInputs.forEach((fileInput) => {
+    if (!(fileInput instanceof HTMLInputElement)) {
+      return;
+    }
+    if (isDisabledForDetection(fileInput)) {
+      return;
+    }
+
+    const anchor = findUploadVisualAnchor(fileInput);
+    if (!anchor) {
+      // No visible anchor anywhere in the ancestor chain — agent has no way
+      // to visually confirm the target. Skip rather than overlay on nothing.
+      return;
+    }
+
+    const rect = getElementRect(anchor);
+    candidates.push({
+      element: fileInput,
+      // Visible anchor element used for overlay rendering — the file input
+      // itself is usually display:none so its own bbox is zero.
+      overlayElement: anchor,
+      type: 'uploadable',
+      signalSource: 'uploadable',
+      rect,
+      area: getElementArea(rect),
+      depth: getDomDepth(fileInput),
+      signalScore: HIGHLIGHT_SIGNAL_SCORE.uploadable || 0,
+    });
+  });
+
+  trace(
+    'uploadable:scan',
+    `fileInputs=${fileInputs.length} anchored=${candidates.length}`,
+  );
+  return candidates;
+}
+
 function collectHighlightCandidates(config, trace, layoutStability) {
   const activeTopLayerRoot = getActiveTopLayerRoot();
   const registry = new Map();
+
+  if (config.elementType === 'uploadable') {
+    // Dedicated fast path: skip the visibility-filtered scan entirely since
+    // file inputs rarely survive it.
+    const uploadables = collectUploadableCandidates(trace);
+    for (const candidate of uploadables) {
+      registry.set(candidate.element, candidate);
+    }
+    const sortedUploadables = Array.from(registry.values()).sort(
+      compareDisplayCandidates,
+    );
+    const uploadableElements = sortedUploadables.map((candidate) =>
+      toInteractiveElement(candidate),
+    );
+    const uploadableCounts = {
+      clickable: 0,
+      scrollable: 0,
+      inputable: 0,
+      selectable: 0,
+      draggable: 0,
+      droppable: 0,
+      uploadable: uploadableElements.length,
+    };
+    trace(
+      'scan:done',
+      `mode=uploadable matched=${uploadableElements.length}`,
+    );
+    return { elements: uploadableElements, counts: uploadableCounts };
+  }
+
+  // Uploadable candidates live outside the visibility-filtered scan. Seed
+  // them into the registry for `any` mode so file inputs (usually hidden)
+  // show up in the default inventory alongside clickables, inputables, etc.
+  if (config.elementType === 'any') {
+    const uploadables = collectUploadableCandidates(trace);
+    for (const candidate of uploadables) {
+      registry.set(candidate.element, candidate);
+    }
+  }
+
   const allElements = getCandidateElementsForScan(
     layoutStability,
     trace,
@@ -2425,6 +2599,7 @@ function collectHighlightCandidates(config, trace, layoutStability) {
     selectable: 0,
     draggable: 0,
     droppable: 0,
+    uploadable: 0,
   };
 
   const elements = prunedCandidates.map((candidate) => {
diff --git a/extension/src/types.ts b/extension/src/types.ts
index 4991a2c..c65f96e 100644
--- a/extension/src/types.ts
+++ b/extension/src/types.ts
@@ -235,6 +235,19 @@ export interface SetSliderValueCommand extends BaseCommand {
   tab_id?: number;
 }
 
+export interface UploadFileCommand extends BaseCommand {
+  type: 'upload_file';
+  /** Element ID of the <input type="file"> from highlight response */
+  element_id: string;
+  /**
+   * Absolute path to the file on the host running the browser. The server
+   * validates existence and readability before dispatching to the extension.
+   */
+  file_path: string;
+  /** Target tab ID (optional - auto-resolved if not provided) */
+  tab_id?: number;
+}
+
 export interface GetElementHtmlCommand extends BaseCommand {
   type: 'get_element_html';
   element_id: string;
@@ -303,6 +316,7 @@ export type Command =
   | SelectElementCommand
   | DragAndDropElementCommand
   | SetSliderValueCommand
+  | UploadFileCommand
   | GetElementHtmlCommand
   | HighlightSingleElementCommand
   | HighlightDropPreviewCommand
@@ -355,6 +369,7 @@ export type ElementType =
   | 'selectable'
   | 'draggable'
   | 'droppable'
+  | 'uploadable'
   | 'any';
 
 export type InteractionHint =
@@ -369,6 +384,7 @@ export interface InteractiveElement {
   interactionHints?: InteractionHint[]; // Extra interaction hints (e.g. swipable carousel region)
   tagName: string; // HTML tag name
   selector: string; // CSS selector to find element
+  overlaySelector?: string; // Optional: selector of a visible anchor element used only for overlay rendering (used for hidden <input type=file> anchored on a label/button)
   html?: string; // Optional: full HTML of the element (captured at highlight time)
   text?: string; // Visible text content
   searchText?: string; // Normalized semantic search text used by keyword filtering
diff --git a/frontend/index.html b/frontend/index.html
index 844188b..dc272b0 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -3733,6 +3733,89 @@
             border-color: rgba(255, 255, 255, 0.35);
         }
 
+        .attached-image-previews {
+            display: flex;
+            flex-wrap: wrap;
+            gap: 8px;
+            padding: 8px 10px;
+            border-radius: 10px;
+            border: 1px dashed rgba(var(--accent-rgb), 0.35);
+            background: rgba(var(--accent-rgb), 0.06);
+        }
+
+        .attached-image-previews[hidden] {
+            display: none;
+        }
+
+        .attached-image-thumb {
+            position: relative;
+            width: 64px;
+            height: 64px;
+            border-radius: 8px;
+            overflow: hidden;
+            border: 1px solid rgba(255, 255, 255, 0.2);
+            background: rgba(0, 0, 0, 0.35);
+            flex-shrink: 0;
+        }
+
+        .attached-image-thumb img {
+            width: 100%;
+            height: 100%;
+            object-fit: cover;
+            display: block;
+        }
+
+        .attached-image-remove {
+            position: absolute;
+            top: 2px;
+            right: 2px;
+            width: 18px;
+            height: 18px;
+            border-radius: 999px;
+            background: rgba(0, 0, 0, 0.7);
+            color: #ffffff;
+            border: none;
+            font-size: 12px;
+            line-height: 1;
+            cursor: pointer;
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+        }
+
+        .attached-image-remove:hover {
+            background: rgba(130, 30, 30, 0.9);
+        }
+
+        .image-attach-button {
+            background: transparent;
+            color: var(--text-secondary);
+            border: 1px solid rgba(255, 255, 255, 0.18);
+            width: 34px;
+            height: 34px;
+            border-radius: 6px;
+            cursor: pointer;
+            font-size: 16px;
+            line-height: 1;
+            margin-right: 6px;
+            display: inline-flex;
+            align-items: center;
+            justify-content: center;
+            transition: all 0.15s ease;
+        }
+
+        .image-attach-button:hover {
+            color: var(--text-primary);
+            border-color: rgba(var(--accent-rgb), 0.45);
+            background: rgba(var(--accent-rgb), 0.1);
+        }
+
+        .input-area-body.drag-active {
+            outline: 2px dashed rgba(var(--accent-rgb), 0.55);
+            outline-offset: 4px;
+            border-radius: 10px;
+        }
+
         .routine-slash-menu {
             position: absolute;
             left: 0;
@@ -5012,14 +5095,9 @@ <h2 class="panel-title">Tell the browser what to do</h2>
                         </button>
                     </div>
 
-                    <div class="quick-prompts">
-                        <button class="quick-prompt" onclick="setCommand('Open https://github.com/softpudding/OpenBrowser and introduce OpenBrowser in a concise way.');">Introduce OpenBrowser</button>
-                        <button class="quick-prompt" onclick="setCommand('Open https://github.com/softpudding/OpenBrowser and star it.');">Star OpenBrowser</button>
-                    </div>
-
                     <div class="input-area">
                         <div class="input-prompt blink">Task</div>
-                        <div class="input-area-body">
+                        <div class="input-area-body" id="input-area-body">
                             <div class="routine-input-card" id="routine-input-card" hidden>
                                 <div class="routine-input-card-info">
                                     <span class="routine-input-card-icon">▶</span>
@@ -5035,9 +5113,18 @@ <h2 class="panel-title">Tell the browser what to do</h2>
                                     onclick="clearStagedRoutine()"
                                 >×</button>
                             </div>
-                            <textarea class="command-input scrollbar" id="command-input" placeholder="Describe the result you want — type / to insert a saved routine" rows="1" autofocus></textarea>
+                            <div class="attached-image-previews" id="attached-image-previews" hidden></div>
+                            <textarea class="command-input scrollbar" id="command-input" placeholder="Describe the result you want…" rows="1" autofocus></textarea>
                             <div class="routine-slash-menu" id="routine-slash-menu" hidden></div>
                         </div>
+                        <input type="file" id="image-file-picker" accept="image/*" multiple hidden>
+                        <button
+                            type="button"
+                            class="image-attach-button"
+                            id="image-attach-button"
+                            title="Attach image (paste, drop, or click)"
+                            onclick="document.getElementById('image-file-picker').click()"
+                        >📎</button>
                         <button class="send-button" id="send-button" onclick="sendCommand()">Run</button>
                         <button class="pause-button" id="pause-button" onclick="pauseConversation()" style="display: none;">Pause</button>
                         <button class="sisyphus-start-button" id="sisyphus-start-btn" onclick="startSisyphus()" style="display: none; background: #224422; border: 1px solid #448844; color: #cccccc; padding: 4px 12px; cursor: pointer; border-radius: 3px; font-size: 11px;">Start Sisyphus</button>
@@ -7451,6 +7538,112 @@ <h1>Sisyphus</h1>
         let routinesEditingId = null;
         let routinesModalPage = 1;
 
+        // ─────────────────────────────────────────────────────────────────
+        // Attached images: paste, drag-drop, or file picker. Shipped as
+        // data URIs alongside the text prompt so the multimodal LLM can see
+        // what the user is describing.
+        // ─────────────────────────────────────────────────────────────────
+
+        const MAX_IMAGE_BYTES = 10 * 1024 * 1024;
+        const MAX_ATTACHED_IMAGES = 8;
+        let attachedImages = [];
+
+        function renderAttachedImagePreviews() {
+            const strip = document.getElementById('attached-image-previews');
+            if (!strip) {
+                return;
+            }
+            strip.innerHTML = '';
+            if (attachedImages.length === 0) {
+                strip.hidden = true;
+                return;
+            }
+            strip.hidden = false;
+            attachedImages.forEach((img, index) => {
+                const thumb = document.createElement('div');
+                thumb.className = 'attached-image-thumb';
+                thumb.title = `${img.name} (${Math.max(1, Math.round(img.sizeBytes / 1024))}KB)`;
+
+                const imgEl = document.createElement('img');
+                imgEl.src = img.dataUri;
+                imgEl.alt = img.name;
+                thumb.appendChild(imgEl);
+
+                const removeBtn = document.createElement('button');
+                removeBtn.type = 'button';
+                removeBtn.className = 'attached-image-remove';
+                removeBtn.textContent = '×';
+                removeBtn.title = 'Remove';
+                removeBtn.addEventListener('click', () => {
+                    attachedImages.splice(index, 1);
+                    renderAttachedImagePreviews();
+                });
+                thumb.appendChild(removeBtn);
+
+                strip.appendChild(thumb);
+            });
+        }
+
+        function addAttachedImageFromFile(file) {
+            if (!file || !file.type || !file.type.startsWith('image/')) {
+                return;
+            }
+            if (attachedImages.length >= MAX_ATTACHED_IMAGES) {
+                addEvent({
+                    type: 'ErrorEvent',
+                    text: `Cannot attach more than ${MAX_ATTACHED_IMAGES} images at once.`,
+                    timestamp: new Date().toISOString(),
+                });
+                return;
+            }
+            if (file.size > MAX_IMAGE_BYTES) {
+                addEvent({
+                    type: 'ErrorEvent',
+                    text: `Image "${file.name || 'untitled'}" is ${(file.size / (1024 * 1024)).toFixed(1)}MB; limit is ${MAX_IMAGE_BYTES / (1024 * 1024)}MB.`,
+                    timestamp: new Date().toISOString(),
+                });
+                return;
+            }
+            const reader = new FileReader();
+            reader.onload = () => {
+                attachedImages.push({
+                    dataUri: reader.result,
+                    mimeType: file.type,
+                    name: file.name || `pasted-image-${Date.now()}.png`,
+                    sizeBytes: file.size,
+                });
+                renderAttachedImagePreviews();
+            };
+            reader.onerror = () => {
+                addEvent({
+                    type: 'ErrorEvent',
+                    text: `Failed to read image "${file.name || 'untitled'}".`,
+                    timestamp: new Date().toISOString(),
+                });
+            };
+            reader.readAsDataURL(file);
+        }
+
+        function clearAttachedImages() {
+            if (attachedImages.length === 0) {
+                return;
+            }
+            attachedImages = [];
+            renderAttachedImagePreviews();
+        }
+
+        function getAttachedImagesPayload() {
+            if (attachedImages.length === 0) {
+                return null;
+            }
+            return attachedImages.map((img) => ({
+                data_uri: img.dataUri,
+                mime_type: img.mimeType,
+                name: img.name,
+                size_bytes: img.sizeBytes,
+            }));
+        }
+
         function buildRoutinePrompt(routine) {
             // The agent receives the Routine markdown verbatim plus a one-line
             // instruction. Keeping the Routine fully visible in the prompt is
@@ -10075,6 +10268,77 @@ <h1>Sisyphus</h1>
                     sendCommand();
                 }
             });
+
+            // Paste an image directly into the prompt (Cmd-V / Ctrl-V on a screenshot)
+            document.getElementById('command-input').addEventListener('paste', function(e) {
+                if (!e.clipboardData || !e.clipboardData.items) {
+                    return;
+                }
+                let handled = false;
+                for (const item of e.clipboardData.items) {
+                    if (item.kind === 'file' && item.type && item.type.startsWith('image/')) {
+                        const file = item.getAsFile();
+                        if (file) {
+                            addAttachedImageFromFile(file);
+                            handled = true;
+                        }
+                    }
+                }
+                if (handled) {
+                    // Prevent the image from being inserted into the textarea as garbage text
+                    e.preventDefault();
+                }
+            });
+
+            // Drag-and-drop an image onto the input area
+            const inputAreaBody = document.getElementById('input-area-body');
+            if (inputAreaBody) {
+                let dragCounter = 0;
+                inputAreaBody.addEventListener('dragenter', function(e) {
+                    if (e.dataTransfer && Array.from(e.dataTransfer.types || []).includes('Files')) {
+                        dragCounter++;
+                        inputAreaBody.classList.add('drag-active');
+                    }
+                });
+                inputAreaBody.addEventListener('dragover', function(e) {
+                    if (e.dataTransfer && Array.from(e.dataTransfer.types || []).includes('Files')) {
+                        e.preventDefault();
+                    }
+                });
+                inputAreaBody.addEventListener('dragleave', function() {
+                    dragCounter = Math.max(0, dragCounter - 1);
+                    if (dragCounter === 0) {
+                        inputAreaBody.classList.remove('drag-active');
+                    }
+                });
+                inputAreaBody.addEventListener('drop', function(e) {
+                    if (!e.dataTransfer || !e.dataTransfer.files || e.dataTransfer.files.length === 0) {
+                        return;
+                    }
+                    e.preventDefault();
+                    dragCounter = 0;
+                    inputAreaBody.classList.remove('drag-active');
+                    for (const file of e.dataTransfer.files) {
+                        if (file.type && file.type.startsWith('image/')) {
+                            addAttachedImageFromFile(file);
+                        }
+                    }
+                });
+            }
+
+            // Paperclip → hidden file picker
+            const imagePicker = document.getElementById('image-file-picker');
+            if (imagePicker) {
+                imagePicker.addEventListener('change', function(e) {
+                    if (!e.target.files) {
+                        return;
+                    }
+                    for (const file of e.target.files) {
+                        addAttachedImageFromFile(file);
+                    }
+                    e.target.value = '';
+                });
+            }
             document.addEventListener('click', function(event) {
                 if (!recordingMenuOpen) {
                     return;
@@ -10538,7 +10802,17 @@ <h1>Sisyphus</h1>
             fetch(url, {
                 method: 'POST',
                 headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({ text: command, cwd: cwd, browser_id: browserId }),
+                body: JSON.stringify((function() {
+                    const body = { text: command, cwd: cwd, browser_id: browserId };
+                    const images = getAttachedImagesPayload();
+                    if (images) {
+                        body.images = images;
+                    }
+                    // Clear previews now that the request is on the wire — keeps
+                    // subsequent messages from silently re-attaching the same image.
+                    clearAttachedImages();
+                    return body;
+                })()),
                 signal: abortController.signal
             }).then(async response => {
                 console.log(`[Frontend] POST response status: ${response.status}, ok: ${response.ok}, content-type: ${response.headers.get('content-type')}`);
diff --git a/server/agent/api.py b/server/agent/api.py
index c3304a6..53aa826 100644
--- a/server/agent/api.py
+++ b/server/agent/api.py
@@ -22,6 +22,41 @@
 logger = logging.getLogger(__name__)
 
 
+def _build_user_message(
+    message_text: str, images: Optional[List[Dict[str, Any]]] = None
+):
+    """Build a user message payload for conversation.send_message().
+
+    When there are no images, returns the plain text string so the SDK's
+    default behavior is preserved. When images are present, returns a
+    multimodal Message with one ImageContent per attached image.
+    """
+    if not images:
+        return message_text
+
+    from openhands.sdk.llm import Message, TextContent, ImageContent
+
+    image_urls = [img["data_uri"] for img in images if img.get("data_uri")]
+    content: List[Any] = [TextContent(text=message_text)]
+    if image_urls:
+        content.append(ImageContent(image_urls=image_urls))
+    return Message(role="user", content=content)
+
+
+def _format_image_history_marker(images: List[Dict[str, Any]]) -> str:
+    """Render a short text marker describing image attachments for history."""
+    parts = []
+    for img in images:
+        name = img.get("name") or "image"
+        size_bytes = img.get("size_bytes")
+        if isinstance(size_bytes, int) and size_bytes > 0:
+            size_kb = max(1, size_bytes // 1024)
+            parts.append(f"[image attached: {name}, {size_kb}KB]")
+        else:
+            parts.append(f"[image attached: {name}]")
+    return "\n".join(parts)
+
+
 async def _stream_sse_events(
     event_queue: Any,
     conversation_id: str,
@@ -138,7 +173,10 @@ async def create_agent_conversation(
 
 
 async def process_agent_message(
-    conversation_id: str, message_text: str, cwd: str = "."
+    conversation_id: str,
+    message_text: str,
+    cwd: str = ".",
+    images: Optional[List[Dict[str, Any]]] = None,
 ) -> AsyncGenerator[str, None]:
     """Process a message and yield SSE events using thread-based execution
 
@@ -146,6 +184,9 @@ async def process_agent_message(
         conversation_id: Conversation ID to process message in
         message_text: Message text to send to agent
         cwd: Working directory for the conversation if creating new (default: current directory)
+        images: Optional list of attached images; each entry is a dict with
+            keys `data_uri`, `mime_type`, `name`, `size_bytes`. When provided,
+            the message is sent to the LLM as a multimodal Message.
     """
     logger.debug(
         f"DEBUG: process_agent_message called with conversation_id={conversation_id}, message='{message_text[:50]}...', cwd={cwd}"
@@ -162,10 +203,13 @@ async def process_agent_message(
         conversation_id, SessionStatus.ACTIVE, increment_message_count=True
     )
 
-    # Save user message for history
+    # Save user message for history (tag image attachments inline for replay)
     try:
+        history_text = message_text
+        if images:
+            history_text = f"{message_text}\n{_format_image_history_marker(images)}"
         session_manager.save_user_message(
-            conversation_id=conversation_id, message_text=message_text
+            conversation_id=conversation_id, message_text=history_text
         )
     except Exception as e:
         logger.warning(f"Failed to save user message: {e}")
@@ -190,7 +234,13 @@ async def process_agent_message(
 
         worker_failed = False
         try:
-            command_queue.put({"agent_message": message_text, "cwd": cwd})
+            command_queue.put(
+                {
+                    "agent_message": message_text,
+                    "cwd": cwd,
+                    "images": images,
+                }
+            )
 
             async for sse_payload in _stream_sse_events(
                 response_queue,
@@ -236,9 +286,11 @@ def run_conversation():
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
 
-            # Send user message to conversation
+            # Send user message to conversation (multimodal when images attached)
             logger.debug(f"DEBUG: Sending message to conversation")
-            conv_state.conversation.send_message(message_text)
+            conv_state.conversation.send_message(
+                _build_user_message(message_text, images)
+            )
 
             # Run the conversation (check if it's async or sync)
             run_method = conv_state.conversation.run
diff --git a/server/agent/prompts/big_model/element_interaction_tool.j2 b/server/agent/prompts/big_model/element_interaction_tool.j2
index 52ef115..5f79721 100644
--- a/server/agent/prompts/big_model/element_interaction_tool.j2
+++ b/server/agent/prompts/big_model/element_interaction_tool.j2
@@ -14,7 +14,7 @@ Use one `element_id` from the current interactive observation to act on the page
 
 ## Interaction Modes
 
-Only `click`, `keyboard_input`, `select`, and `drag_and_drop` require confirmation before execution. `hover`, `scroll`, `swipe`, and `set_slider` execute immediately and return the default `highlight` `element_type: "any"` page 1 screenshot of the new page state.
+Only `click`, `keyboard_input`, `select`, and `drag_and_drop` require confirmation before execution. `hover`, `scroll`, `swipe`, `set_slider`, and `upload_file` execute immediately and return the default `highlight` `element_type: "any"` page 1 screenshot of the new page state.
 
 ## Visual Color System - YELLOW Stage
 
@@ -157,6 +157,19 @@ Set a slider to a target value. Works on three kinds of slider:
 
 Use `set_slider` for any element marked with the `slidable` interaction hint — do not try to click or drag the thumb manually.
 
+### upload_file
+
+Attach a local file to an `<input type="file">` element. Bypasses the native OS file picker — do not try to `click` the file input first.
+
+```json
+{ "action": "upload_file", "element_id": "F7K", "file_path": "/tmp/resume.pdf", "tab_id": 123 }
+// → Executes immediately and returns the default `highlight` `element_type: "any"` page 1 screenshot
+```
+
+Use this only on elements returned with the `uploadable` hint. The target is always the underlying `<input type="file">` even when the visible control is a styled button or label.
+
+`file_path` must be an absolute path that exists on the host running the browser. The agent is responsible for producing or locating that file beforehand (e.g. writing it to `/tmp/...` with a terminal tool, or using a path the user pasted in plain language). Relative paths and missing files are rejected with a clear error.
+
 ### select
 
 Select an option from a native `<select>` dropdown element by its visual ID.
@@ -203,6 +216,7 @@ Completed `confirm_*` actions return the default `highlight` `element_type: "any
 - For scrollable pages or regions, use `scroll`
 - If the likely target is already visible but poorly positioned, use `scroll` before more discovery or before clicking.
 - For carousel / gallery / swiper regions marked `swipable`, use `swipe`
+- For elements marked `uploadable` (file inputs), use `upload_file` with an absolute `file_path` — do not try to `click` them first.
 - If a returned element is marked `draggable`, prefer `drag_and_drop` over `click`. To find valid drop targets, use `highlight element_type: "droppable"`.
 - If a target region is marked `scrollable` or `swipable`, do not replace that affordance with guessed navigation buttons or guessed controls first
 
@@ -210,7 +224,7 @@ Completed `confirm_*` actions return the default `highlight` `element_type: "any
 
 - `click`, `keyboard_input`, and `select` return a YELLOW preview screenshot before execution.
 - `drag_and_drop` returns a zoomed preview of the container with inner elements highlighted.
-- `hover`, `scroll`, `swipe`, `set_slider`, and completed `confirm_*` actions return the default `highlight` `element_type: "any"` page 1 screenshot for the new page state.
+- `hover`, `scroll`, `swipe`, `set_slider`, `upload_file`, and completed `confirm_*` actions return the default `highlight` `element_type: "any"` page 1 screenshot for the new page state.
 
 ## Error Handling
 
diff --git a/server/agent/prompts/big_model/highlight_tool.j2 b/server/agent/prompts/big_model/highlight_tool.j2
index 116d1b5..cd239aa 100644
--- a/server/agent/prompts/big_model/highlight_tool.j2
+++ b/server/agent/prompts/big_model/highlight_tool.j2
@@ -40,11 +40,12 @@ Build or extend the interactive-element inventory for the current page state.
 { "element_type": "selectable" }     // Native <select> elements
 { "element_type": "draggable" }      // Drag sources (cards, handles, sliders)
 { "element_type": "droppable" }      // Drop targets (columns, zones, tracks)
+{ "element_type": "uploadable" }     // File inputs (use upload_file to attach a local file)
 { "keywords": ["Continue with Email"] } // Exact observed readable text only
 ```
 
 **Parameters**:
-- `element_type`: Single type to highlight - `"any"` (default), `"scrollable"`, `"inputable"`, `"selectable"`, `"draggable"`, or `"droppable"`
+- `element_type`: Single type to highlight - `"any"` (default), `"scrollable"`, `"inputable"`, `"selectable"`, `"draggable"`, `"droppable"`, or `"uploadable"`
 - `page`: Page number for pagination (1-indexed, default 1). Ignored when `keywords` is provided.
 - `keywords`: Exact literal text already visible on the target itself in the current screenshot or current highlight HTML. Use this only to accelerate a known text match, not to guess controls. When provided, all matching elements are returned without pagination.
 
@@ -70,6 +71,7 @@ Build or extend the interactive-element inventory for the current page state.
 - If a returned element is marked `scrollable`, prefer `scroll` on that region when your goal is to reveal more content inside it.
 - If a returned element is marked `draggable`, prefer `drag_and_drop` over `click`. To find valid drop targets, use `element_type: "droppable"`.
 - If a returned element is marked `slidable`, use `set_slider` — do not click or drag the slider thumb manually. This applies to video progress bars, volume controls, range inputs, etc.
+- If a returned element is marked `uploadable`, use `upload_file` with an absolute `file_path`. Do not `click` a file input — clicking it would open a native OS picker the agent cannot drive.
 - For drag-and-drop tasks (kanban boards, reorderable lists): first identify drag sources (`draggable` hint in `any` or `element_type: "draggable"`), then find drop targets (`element_type: "droppable"`), then use `drag_and_drop` with `element_id` (source) and `target_element_id` (drop zone).
 - For text-entry targets, if `any` was not enough you may narrow to `inputable`. Once you pick an input target, always `click` it first and complete that confirmation before `keyboard_input`.
 - If a later action says the document changed, the target identity changed, or the cached element is stale, stop using the old IDs and rebuild with `highlight`.
diff --git a/server/agent/prompts/small_model/element_interaction_tool.j2 b/server/agent/prompts/small_model/element_interaction_tool.j2
index bce2490..9fb7ea6 100644
--- a/server/agent/prompts/small_model/element_interaction_tool.j2
+++ b/server/agent/prompts/small_model/element_interaction_tool.j2
@@ -89,6 +89,14 @@ These are content directions, not hand or finger movement directions. Do not rei
 
 Use this for drag-and-drop cards, reorderable lists, or sortable items. Returns a zoomed preview of the container with inner elements; use `confirm_drag_and_drop` to drop at end or with `relative_to`/`position` for precise placement.
 
+### upload_file
+
+```json
+{ "action": "upload_file", "element_id": "F7K", "file_path": "/tmp/resume.pdf", "tab_id": 123 }
+```
+
+Use on elements with the `uploadable` hint (file inputs). `file_path` must be an absolute path on the host. Bypasses the native file picker — do not `click` the input first.
+
 ### set_slider
 
 ```json
@@ -109,6 +117,7 @@ Use this for any element with the `slidable` hint — native `<input type=range>
 - If the page or target region is scrollable and your goal is to reveal more content, use `scroll`
 - If the likely target is already visible but poorly positioned, use `scroll` before more highlight pagination or before clicking.
 - For carousel / gallery / swiper regions marked `swipable`, use `swipe`
+- For elements marked `uploadable` (file inputs), use `upload_file` with an absolute `file_path`
 - If a returned element is marked `draggable`, prefer `drag_and_drop` over `click`. To find valid drop targets, use `highlight element_type: "droppable"`.
 - If a target region is marked `scrollable` or `swipable`, do not replace that affordance with guessed navigation buttons or guessed controls first
 
diff --git a/server/agent/prompts/small_model/highlight_tool.j2 b/server/agent/prompts/small_model/highlight_tool.j2
index 7599aec..1866781 100644
--- a/server/agent/prompts/small_model/highlight_tool.j2
+++ b/server/agent/prompts/small_model/highlight_tool.j2
@@ -25,6 +25,7 @@ Build or extend the interactive-element inventory for the current page state.
 { "element_type": "selectable" }
 { "element_type": "draggable" }
 { "element_type": "droppable" }
+{ "element_type": "uploadable" }
 { "page": 2 }
 ```
 
@@ -36,11 +37,12 @@ Build or extend the interactive-element inventory for the current page state.
 - If the likely target is already partly visible or clipped, fix geometry with `scroll` before more pagination.
 - If page 1 missed the target on the same unchanged page state and it is not already partly visible, your default next step is the next page in the same mode.
 - If dense UI or collision-aware label placement may have split nearby controls across pages, keep paginating the same mode before narrowing or switching strategies.
-- Narrow to `inputable`, `scrollable`, `selectable`, `draggable`, or `droppable` only when the task directly targets that affordance and `any` was not enough.
+- Narrow to `inputable`, `scrollable`, `selectable`, `draggable`, `droppable`, or `uploadable` only when the task directly targets that affordance and `any` was not enough.
 - If a control is icon-only or the text is not clearly readable, continue pagination instead of guessing labels such as `settings`, `gear`, `bell`, `next`, `prev`, or `close`.
 - If highlight shows `swipable`, use `swipe`.
 - If a returned element is marked `draggable`, prefer `drag_and_drop` over `click`. To find valid drop targets, use `element_type: "droppable"`.
 - If a returned element is marked `slidable`, use `set_slider` — do not click or drag the slider.
+- If a returned element is marked `uploadable`, use `upload_file` with an absolute `file_path`. Do not `click` it.
 - If a returned element is marked `scrollable`, prefer `scroll` on that region when your goal is to reveal more content inside it.
 - For drag-and-drop tasks (kanban boards, reorderable lists): first identify drag sources (`draggable` hint in `any` or `element_type: "draggable"`), then find drop targets (`element_type: "droppable"`), then use `drag_and_drop` with `element_id` (source) and `target_element_id` (drop zone).
 - For text-entry targets, always `click` first and complete that confirmation before `keyboard_input`.
diff --git a/server/agent/tools/browser_executor.py b/server/agent/tools/browser_executor.py
index 8577f1c..26b3f35 100644
--- a/server/agent/tools/browser_executor.py
+++ b/server/agent/tools/browser_executor.py
@@ -38,6 +38,7 @@
     SelectElementCommand,
     DragAndDropElementCommand,
     SetSliderValueCommand,
+    UploadFileCommand,
     HighlightDropPreviewCommand,
 )
 
@@ -546,6 +547,26 @@ def _execute_element_interaction_action(
                 element_id=action.element_id,
             )
 
+        elif action_type == "upload_file":
+            if not action.element_id:
+                raise ValueError("upload_file requires element_id parameter")
+            if not action.file_path:
+                raise ValueError(
+                    "upload_file requires file_path (absolute path on the server host)"
+                )
+            command = UploadFileCommand(
+                element_id=action.element_id,
+                file_path=action.file_path,
+                conversation_id=self.conversation_id,
+                tab_id=action.tab_id,
+            )
+            result_dict = self._execute_element_command(command, "upload file")
+            return self._build_observation_from_result(
+                result_dict,
+                f"Uploaded file to {action.element_id}: {action.file_path}",
+                element_id=action.element_id,
+            )
+
         elif action_type == "keyboard_input":
             if not action.element_id:
                 raise ValueError("keyboard_input requires element_id parameter")
diff --git a/server/agent/tools/element_interaction_tool.py b/server/agent/tools/element_interaction_tool.py
index 8d498f2..28b4e3c 100644
--- a/server/agent/tools/element_interaction_tool.py
+++ b/server/agent/tools/element_interaction_tool.py
@@ -41,16 +41,17 @@ class ElementInteractionAction(OpenBrowserAction):
         "select",
         "drag_and_drop",
         "set_slider",
+        "upload_file",
         "confirm_click",
         "confirm_keyboard_input",
         "confirm_select",
         "confirm_drag_and_drop",
     ] = Field(
-        description="Element interaction action (click, keyboard_input, select, and drag_and_drop require confirm_* follow-up; confirm_* executes the current pending confirmation; hover/scroll/swipe/set_slider execute directly)"
+        description="Element interaction action (click, keyboard_input, select, and drag_and_drop require confirm_* follow-up; confirm_* executes the current pending confirmation; hover/scroll/swipe/set_slider/upload_file execute directly)"
     )
     element_id: Optional[str] = Field(
         default=None,
-        description="Element ID (short opaque string) from highlight. Required for click, hover, element scroll/swipe, keyboard_input, select, drag_and_drop (source), and set_slider. Ignored for confirm_* actions.",
+        description="Element ID (short opaque string) from highlight. Required for click, hover, element scroll/swipe, keyboard_input, select, drag_and_drop (source), set_slider, and upload_file. Ignored for confirm_* actions.",
     )
     direction: Optional[Literal["up", "down", "left", "right", "next", "prev"]] = Field(
         default="down",
@@ -117,6 +118,14 @@ class ElementInteractionAction(OpenBrowserAction):
         le=40,
         description="Number of intermediate mousemove steps for drag_and_drop (2-40).",
     )
+    file_path: Optional[str] = Field(
+        default=None,
+        description=(
+            "For `upload_file`: absolute path to the local file to attach to "
+            "the target <input type=file>. Must exist on the host running the "
+            "browser. Required for upload_file; ignored otherwise."
+        ),
+    )
     tab_id: Optional[int] = Field(
         default=None,
         description="Tab ID (optional, uses active tab if not specified). confirm_* actions use the tab stored in the pending confirmation.",
diff --git a/server/agent/tools/highlight_tool.py b/server/agent/tools/highlight_tool.py
index 26567b1..773e809 100644
--- a/server/agent/tools/highlight_tool.py
+++ b/server/agent/tools/highlight_tool.py
@@ -34,7 +34,7 @@ class BaseHighlightAction(OpenBrowserAction):
 
     element_type: str = Field(
         default="any",
-        description="Single element type to highlight for agent-visible guidance: any/scrollable/inputable/selectable/draggable/droppable. Defaults to 'any'.",
+        description="Single element type to highlight for agent-visible guidance: any/scrollable/inputable/selectable/draggable/droppable/uploadable. Defaults to 'any'.",
     )
     page: int = Field(
         default=1,
diff --git a/server/api/routes/agent.py b/server/api/routes/agent.py
index 9211e1d..213e72a 100644
--- a/server/api/routes/agent.py
+++ b/server/api/routes/agent.py
@@ -3,9 +3,15 @@
 import asyncio
 import json
 import logging
+from typing import Any, Dict, List, Optional
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import StreamingResponse
 
+# Max raw bytes accepted per attached image (pre-base64). The frontend rejects
+# at the same threshold; this is a defense against spoofed clients.
+MAX_IMAGE_BYTES = 10 * 1024 * 1024
+MAX_IMAGES_PER_MESSAGE = 8
+
 from server.agent.agent import (
     agent_manager,
     process_agent_message,
@@ -27,6 +33,73 @@
 logger = logging.getLogger(__name__)
 
 
+def _validate_images(
+    raw_images: Any,
+) -> Optional[List[Dict[str, Any]]]:
+    """Validate and normalize the optional `images` field on a message POST.
+
+    Each entry must be a dict with a `data_uri` starting with `data:image/`
+    and decoded payload size ≤ MAX_IMAGE_BYTES. Returns a normalized list
+    with `data_uri`, `mime_type`, `name`, and `size_bytes` set, or None when
+    no images were provided.
+    """
+    if raw_images is None:
+        return None
+    if not isinstance(raw_images, list):
+        raise HTTPException(
+            status_code=400, detail="`images` must be a list when provided"
+        )
+    if len(raw_images) > MAX_IMAGES_PER_MESSAGE:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Too many images (max {MAX_IMAGES_PER_MESSAGE})",
+        )
+    normalized: List[Dict[str, Any]] = []
+    for i, entry in enumerate(raw_images):
+        if not isinstance(entry, dict):
+            raise HTTPException(
+                status_code=400, detail=f"images[{i}] must be an object"
+            )
+        data_uri = entry.get("data_uri")
+        if not isinstance(data_uri, str) or not data_uri.startswith("data:image/"):
+            raise HTTPException(
+                status_code=400,
+                detail=f"images[{i}].data_uri must be a data:image/… URI",
+            )
+        # Parse mime + base64 payload from the data URI
+        try:
+            header, payload = data_uri.split(",", 1)
+            if ";base64" not in header:
+                raise ValueError("only base64-encoded data URIs are accepted")
+            mime_type = header[len("data:") : header.index(";base64")]
+            decoded_size = (len(payload) * 3) // 4  # approximate; exact enough
+            if decoded_size > MAX_IMAGE_BYTES:
+                raise HTTPException(
+                    status_code=400,
+                    detail=(
+                        f"images[{i}] exceeds {MAX_IMAGE_BYTES // (1024 * 1024)}MB "
+                        f"(got ~{decoded_size // (1024 * 1024)}MB)"
+                    ),
+                )
+        except HTTPException:
+            raise
+        except Exception as exc:
+            raise HTTPException(
+                status_code=400, detail=f"images[{i}] is not a valid data URI: {exc}"
+            )
+        normalized.append(
+            {
+                "data_uri": data_uri,
+                "mime_type": mime_type or entry.get("mime_type"),
+                "name": entry.get("name") or f"image-{i + 1}",
+                "size_bytes": entry.get("size_bytes")
+                if isinstance(entry.get("size_bytes"), int)
+                else decoded_size,
+            }
+        )
+    return normalized
+
+
 def _require_valid_browser_id(browser_id: str | None) -> str:
     """Require a valid, registered browser UUID."""
     if not browser_id:
@@ -183,7 +256,11 @@ async def agent_messages_stream(conversation_id: str, request: Request):
     - POST: Send a message and get SSE stream response
     """
 
-    async def event_generator(message_text: str = None, cwd: str = "."):
+    async def event_generator(
+        message_text: str = None,
+        cwd: str = ".",
+        images: Optional[List[Dict[str, Any]]] = None,
+    ):
         """Generate SSE events for the agent conversation"""
         try:
             # If no message text provided, this is a GET request - just open stream
@@ -204,7 +281,7 @@ async def event_generator(message_text: str = None, cwd: str = "."):
                 )
                 event_count = 0
                 async for sse_event in process_agent_message(
-                    conversation_id, message_text, cwd
+                    conversation_id, message_text, cwd, images=images
                 ):
                     event_count += 1
                     logger.debug(
@@ -268,9 +345,10 @@ async def event_generator(message_text: str = None, cwd: str = "."):
 
             # Extract cwd parameter with default value
             cwd = message_data.get("cwd", ".")
+            images = _validate_images(message_data.get("images"))
 
             return StreamingResponse(
-                event_generator(message_data["text"], cwd),
+                event_generator(message_data["text"], cwd, images),
                 media_type="text/event-stream",
                 headers=create_sse_response_headers(),
             )
diff --git a/server/core/browser_executor_bundle.py b/server/core/browser_executor_bundle.py
index e40c2ed..6837232 100644
--- a/server/core/browser_executor_bundle.py
+++ b/server/core/browser_executor_bundle.py
@@ -244,11 +244,14 @@ async def execute_agent_message(
         self,
         message_text: str,
         event_queue: Any,
+        images: list[dict[str, Any]] | None = None,
     ) -> dict[str, Any]:
         """Execute a conversation turn inside the worker process.
 
         Events are streamed back to the main process via ``event_queue`` as
         ``SSEEvent`` instances so the HTTP layer can forward them unchanged.
+        When ``images`` is provided, the user message is sent as a multimodal
+        ``Message`` with one ``ImageContent`` per attachment.
         """
         if not self.state.initialized:
             raise RuntimeError(
@@ -276,7 +279,20 @@ async def execute_agent_message(
         conv_state.visualizer.set_event_queue(event_queue)
 
         try:
-            conv_state.conversation.send_message(message_text)
+            if images:
+                from openhands.sdk.llm import Message, TextContent, ImageContent
+
+                image_urls = [
+                    img["data_uri"] for img in images if img.get("data_uri")
+                ]
+                content: list[Any] = [TextContent(text=message_text)]
+                if image_urls:
+                    content.append(ImageContent(image_urls=image_urls))
+                conv_state.conversation.send_message(
+                    Message(role="user", content=content)
+                )
+            else:
+                conv_state.conversation.send_message(message_text)
 
             run_method = conv_state.conversation.run
             if inspect.iscoroutinefunction(run_method):
diff --git a/server/core/process_manager.py b/server/core/process_manager.py
index 3e709f5..46f7550 100644
--- a/server/core/process_manager.py
+++ b/server/core/process_manager.py
@@ -127,6 +127,7 @@ def _conversation_worker(
                         bundle.execute_agent_message(
                             message_text=message["agent_message"],
                             event_queue=response_queue,
+                            images=message.get("images"),
                         )
                     )
 
diff --git a/server/core/processor.py b/server/core/processor.py
index 1a75665..c15a9d7 100644
--- a/server/core/processor.py
+++ b/server/core/processor.py
@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import json
+import os
 from typing import Dict, Optional, Any, List
 from datetime import datetime
 
@@ -33,6 +34,7 @@
     RecordingControlCommand,
     DragAndDropElementCommand,
     SetSliderValueCommand,
+    UploadFileCommand,
     HighlightDropPreviewCommand,
 )
 from server.websocket.manager import ws_manager
@@ -260,6 +262,8 @@ async def execute(self, command: Command) -> CommandResponse:
                 return await self._execute_recording_control(command)
             elif isinstance(command, HighlightDropPreviewCommand):
                 return await self._execute_highlight_drop_preview(command)
+            elif isinstance(command, UploadFileCommand):
+                return await self._execute_upload_file(command)
             else:
                 raise ValueError(f"Unknown command type: {command.type}")
 
@@ -454,6 +458,38 @@ async def _execute_highlight_drop_preview(
         """Highlight inner elements of a drop container for drag-and-drop 2PC"""
         return await self._send_prepared_command(command)
 
+    async def _execute_upload_file(
+        self, command: UploadFileCommand
+    ) -> CommandResponse:
+        """Attach a local file to an <input type=file> via CDP.
+
+        Validates the path server-side before dispatching to the extension,
+        so a bad path returns an observation the agent can react to instead
+        of failing opaquely inside CDP.
+        """
+        if not os.path.isabs(command.file_path):
+            return CommandResponse(
+                success=False,
+                command_id=command.command_id,
+                error=(
+                    f"file_path must be absolute, got: {command.file_path!r}. "
+                    "Pass the full path (e.g. /tmp/file.png)."
+                ),
+            )
+        if not os.path.isfile(command.file_path):
+            return CommandResponse(
+                success=False,
+                command_id=command.command_id,
+                error=f"File not found: {command.file_path}",
+            )
+        if not os.access(command.file_path, os.R_OK):
+            return CommandResponse(
+                success=False,
+                command_id=command.command_id,
+                error=f"File not readable: {command.file_path}",
+            )
+        return await self._send_prepared_command(command)
+
     async def _execute_select_element(
         self, command: SelectElementCommand
     ) -> CommandResponse:
diff --git a/server/models/commands.py b/server/models/commands.py
index 592d3d9..1c4377c 100644
--- a/server/models/commands.py
+++ b/server/models/commands.py
@@ -273,7 +273,7 @@ class HighlightElementsCommand(BaseCommand):
     type: Literal["highlight_elements"] = "highlight_elements"
     element_type: Optional[str] = Field(
         default="any",
-        description="Single element type to highlight for agent-visible guidance: 'any', 'scrollable', 'inputable', 'selectable', 'draggable', or 'droppable'",
+        description="Single element type to highlight for agent-visible guidance: 'any', 'scrollable', 'inputable', 'selectable', 'draggable', 'droppable', or 'uploadable'",
     )
     page: Optional[int] = Field(
         default=1,
@@ -465,6 +465,28 @@ class DragAndDropElementCommand(BaseCommand):
     )
 
 
+class UploadFileCommand(BaseCommand):
+    """Attach a file to an <input type="file"> element via CDP.
+
+    Bypasses the native OS file picker by calling DOM.setFileInputFiles
+    directly against the file input node. The browser must be able to
+    read file_path on its local filesystem, so for v1 the server and
+    Chrome must run on the same host.
+
+    tab_id is optional and will be auto-resolved to the current managed tab if not provided.
+    """
+
+    type: Literal["upload_file"] = "upload_file"
+    element_id: str = Field(description="Element ID of an uploadable <input type=file>")
+    file_path: str = Field(
+        description="Absolute path to the file to upload. Must exist on the host running the browser."
+    )
+    tab_id: Optional[int] = Field(
+        default=None,
+        description="Target tab ID (optional, auto-resolved if not provided)",
+    )
+
+
 class SetSliderValueCommand(BaseCommand):
     """Set the value of a native <input type=range> slider directly.
 
@@ -593,6 +615,7 @@ class TabsResponse(CommandResponse):
     RecordingControlCommand,
     DragAndDropElementCommand,
     SetSliderValueCommand,
+    UploadFileCommand,
 ]
 
 
@@ -630,6 +653,7 @@ def parse_command(data: dict) -> Command:
         "recording_control": RecordingControlCommand,
         "drag_and_drop_element": DragAndDropElementCommand,
         "set_slider_value": SetSliderValueCommand,
+        "upload_file": UploadFileCommand,
     }
 
     if cmd_type not in command_map:
diff --git a/server/tests/unit/test_agent_api_multiprocess.py b/server/tests/unit/test_agent_api_multiprocess.py
index cb820cc..79d4c49 100644
--- a/server/tests/unit/test_agent_api_multiprocess.py
+++ b/server/tests/unit/test_agent_api_multiprocess.py
@@ -47,6 +47,7 @@ async def test_process_agent_message_uses_worker_queues() -> None:
     assert command_queue.get_nowait() == {
         "agent_message": "hello worker",
         "cwd": "/tmp/workspace",
+        "images": None,
     }
     assert any("event: agent_event" in payload for payload in sse_payloads)
     assert any("event: complete" in payload for payload in sse_payloads)
diff --git a/server/tests/unit/test_api_uuid.py b/server/tests/unit/test_api_uuid.py
index fc115b1..7c756bb 100644
--- a/server/tests/unit/test_api_uuid.py
+++ b/server/tests/unit/test_api_uuid.py
@@ -281,7 +281,10 @@ def test_message_post_uses_bound_browser_id_when_request_omits_it(
         assert response.status_code == 200
         assert "event: message" in body
         mock_process_agent_message.assert_called_once_with(
-            "test-conversation", "hello from bound conversation", "."
+            "test-conversation",
+            "hello from bound conversation",
+            ".",
+            images=None,
         )
 
     def test_message_post_rejects_rebinding_conversation_to_different_browser(

From f34f60b83de882159b33dd6b1859cf962d9d122e Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Thu, 16 Apr 2026 21:40:16 +0800
Subject: [PATCH 02/12] skill(open-browser): document and support image
 attachments

Adds a --image PATH flag to send_task.py that reads a local image,
base64-encodes it, and sends it alongside the user message as a data
URI. Matches the server's 10MB / 8-image limits so the agent sees the
same 400s the UI would.

SKILL.md gains an "Attaching images" section covering the typical
multimodal use cases (visual regression, reproducing a bug from a
screenshot, asset matching). api_reference.md documents the
`images` field on POST /agent/conversations/{id}/messages with field
descriptions, limits, and a curl example.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 skill/claude/open-browser/SKILL.md            | 31 ++++++++
 .../open-browser/references/api_reference.md  | 32 +++++++++
 .../claude/open-browser/scripts/send_task.py  | 72 ++++++++++++++++++-
 3 files changed, 132 insertions(+), 3 deletions(-)

diff --git a/skill/claude/open-browser/SKILL.md b/skill/claude/open-browser/SKILL.md
index ad75d2e..1130574 100644
--- a/skill/claude/open-browser/SKILL.md
+++ b/skill/claude/open-browser/SKILL.md
@@ -103,6 +103,37 @@ The legacy `--background --output /tmp/openbrowser.log` flag still
 works as a fallback for shell-only environments, but it should not be
 your default in Claude Code.
 
+## Attaching images to a task
+
+OpenBrowser uses multimodal LLMs, so you can send an image (screenshot,
+UI mockup, reference photo) alongside the text prompt. Use this when the
+task is "recreate this design", "why does my UI look wrong compared to
+this screenshot", or "find the element that matches this picture".
+
+Pass `--image PATH` once per image. Images are read from disk, base64
+encoded, and sent as data URIs — no upload endpoint or static server is
+required. Limit: 10 MB per image, up to 8 images per message.
+
+```bash
+python3 skill/claude/open-browser/scripts/send_task.py \
+  "Open the local dashboard and tell me which section looks different from this screenshot." \
+  --image /tmp/reference.png \
+  --chrome-uuid "$OPENBROWSER_CHROME_UUID"
+```
+
+Typical use cases:
+- **Visual regression check**: screenshot the expected UI, send it with
+  "compare this to the current page at http://localhost:3000 and list
+  differences".
+- **Reproducing a bug from a user report**: drop in the user's
+  screenshot and ask "navigate to the page shown here and confirm you
+  see the same error".
+- **Asset-matching**: send a design mockup and ask the agent to pick
+  the closest live element from the current page.
+
+The conversation history saves an `[image attached: name, NkB]` marker
+(not the bytes) so replays don't balloon to megabytes.
+
 ## Follow-up turns on the same browser session
 
 To send a second instruction to the same conversation (so the agent
diff --git a/skill/claude/open-browser/references/api_reference.md b/skill/claude/open-browser/references/api_reference.md
index 9d269f9..8eb0a77 100644
--- a/skill/claude/open-browser/references/api_reference.md
+++ b/skill/claude/open-browser/references/api_reference.md
@@ -64,6 +64,38 @@ curl -N -X POST http://127.0.0.1:8765/agent/conversations/CONVERSATION_ID/messag
 `browser_id` is required for actual browser control unless the
 conversation is already bound to a browser UUID.
 
+## Attaching images to the user message
+
+The `/messages` endpoint accepts an optional `images` array so a
+multimodal LLM can see what the user is describing. Each entry is a
+dict with these fields:
+
+| field        | required | notes                                        |
+|--------------|----------|----------------------------------------------|
+| `data_uri`   | yes      | `data:image/...;base64,...` — must start with `data:image/` |
+| `mime_type`  | no       | Informational; the server parses it from the URI as well |
+| `name`       | no       | Used in the history marker                    |
+| `size_bytes` | no       | Populated by helper scripts                   |
+
+Server limits (kept in sync with the frontend): at most 8 images per
+message, each at most 10 MB of raw bytes pre-encoding. Exceeding either
+returns HTTP 400 with a descriptive `detail`.
+
+```bash
+DATA_URI="data:image/png;base64,$(base64 -i /tmp/screenshot.png | tr -d '\n')"
+curl -N -X POST http://127.0.0.1:8765/agent/conversations/CONVERSATION_ID/messages \
+  -H 'Content-Type: application/json' \
+  -H 'Accept: text/event-stream' \
+  -d "$(jq -n --arg text "Compare the current page to this screenshot." \
+               --arg cwd "." \
+               --arg uuid "$OPENBROWSER_CHROME_UUID" \
+               --arg uri "$DATA_URI" \
+               '{text: $text, cwd: $cwd, browser_id: $uuid, images: [{data_uri: $uri, name: "screenshot.png"}]}')"
+```
+
+The `send_task.py` helper handles the base64 + data URI packaging for
+you — prefer `--image PATH` over a hand-rolled curl unless you need API-level control.
+
 ## Reusing a conversation across turns
 
 To send a follow-up turn to the same browser context (preserving the
diff --git a/skill/claude/open-browser/scripts/send_task.py b/skill/claude/open-browser/scripts/send_task.py
index 76f290d..8588400 100644
--- a/skill/claude/open-browser/scripts/send_task.py
+++ b/skill/claude/open-browser/scripts/send_task.py
@@ -17,13 +17,20 @@
 from __future__ import annotations
 
 import argparse
+import base64
 import json
+import mimetypes
 import os
 import subprocess
 import sys
+from pathlib import Path
 from urllib.error import URLError
 from urllib.request import Request, urlopen
 
+# Per-image cap enforced by the server (see server/api/routes/agent.py).
+# Keep this in sync: the server will reject larger payloads with 400.
+MAX_IMAGE_BYTES = 10 * 1024 * 1024
+
 
 def request_json(
     url: str,
@@ -158,6 +165,40 @@ def format_event(event_type: str, data: dict, *, show_system_prompt: bool) -> No
     )
 
 
+def load_image_as_payload(path: str) -> dict:
+    """Read a local image file and package it for the OpenBrowser API.
+
+    The server accepts data URIs so the frontend and server don't need to
+    share a filesystem. We base64-encode the raw bytes here and tag the
+    entry with mime/name/size so the server's size cap and history marker
+    both work.
+    """
+    image_path = Path(path).expanduser().resolve()
+    if not image_path.is_file():
+        raise FileNotFoundError(f"Image not found: {image_path}")
+
+    data = image_path.read_bytes()
+    if len(data) > MAX_IMAGE_BYTES:
+        raise ValueError(
+            f"Image {image_path} is {len(data) / (1024 * 1024):.1f}MB; "
+            f"server limit is {MAX_IMAGE_BYTES // (1024 * 1024)}MB per image."
+        )
+
+    mime_type, _ = mimetypes.guess_type(image_path.name)
+    if not mime_type or not mime_type.startswith("image/"):
+        raise ValueError(
+            f"{image_path} does not look like an image (mime={mime_type!r})."
+        )
+
+    encoded = base64.b64encode(data).decode("ascii")
+    return {
+        "data_uri": f"data:{mime_type};base64,{encoded}",
+        "mime_type": mime_type,
+        "name": image_path.name,
+        "size_bytes": len(data),
+    }
+
+
 def stream_task(
     base_url: str,
     conversation_id: str,
@@ -166,12 +207,15 @@ def stream_task(
     chrome_uuid: str,
     *,
     show_system_prompt: bool,
+    images: list[dict] | None = None,
 ) -> None:
+    body: dict = {"text": task, "cwd": cwd, "browser_id": chrome_uuid}
+    if images:
+        body["images"] = images
+
     request = Request(
         f"{base_url}/agent/conversations/{conversation_id}/messages",
-        data=json.dumps({"text": task, "cwd": cwd, "browser_id": chrome_uuid}).encode(
-            "utf-8"
-        ),
+        data=json.dumps(body).encode("utf-8"),
         headers={
             "Content-Type": "application/json",
             "Accept": "text/event-stream",
@@ -233,6 +277,8 @@ def start_background_process(args: argparse.Namespace) -> int:
         command.extend(["--conversation-id", args.conversation_id])
     if args.show_system_prompt:
         command.append("--show-system-prompt")
+    for image_path in args.image or []:
+        command.extend(["--image", image_path])
 
     with open(args.output, "a", encoding="utf-8") as log_file:
         process = subprocess.Popen(
@@ -278,6 +324,17 @@ def main() -> int:
         action="store_true",
         help="Print the SystemPromptEvent (suppressed by default to keep logs readable).",
     )
+    parser.add_argument(
+        "--image",
+        action="append",
+        metavar="PATH",
+        help=(
+            "Attach an image (PNG/JPEG/GIF/WebP) to the user message. "
+            "Repeat the flag to attach multiple images. Images are read "
+            "from disk, base64-encoded, and sent as data URIs. Max 10MB "
+            "per image, up to 8 images per message."
+        ),
+    )
     parser.add_argument(
         "--background",
         action="store_true",
@@ -348,6 +405,14 @@ def main() -> int:
             print(f"Browser UUID validation failed: {message}", file=sys.stderr)
             return 1
 
+        images: list[dict] | None = None
+        if args.image:
+            try:
+                images = [load_image_as_payload(p) for p in args.image]
+            except (FileNotFoundError, ValueError) as exc:
+                print(str(exc), file=sys.stderr)
+                return 2
+
         if args.conversation_id:
             conversation_id = args.conversation_id
         else:
@@ -360,6 +425,7 @@ def main() -> int:
             args.cwd,
             args.chrome_uuid,
             show_system_prompt=args.show_system_prompt,
+            images=images,
         )
         return 0
     except URLError as exc:

From f239cf197e6440977b67c5c623a23c5615db2e44 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Sat, 18 Apr 2026 10:21:42 +0800
Subject: [PATCH 03/12] chore: apply pre-commit formatting (black + prettier)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 extension/src/commands/highlight-detection.injected.js | 5 +----
 server/api/routes/agent.py                             | 8 +++++---
 server/core/browser_executor_bundle.py                 | 4 +---
 server/core/processor.py                               | 4 +---
 4 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/extension/src/commands/highlight-detection.injected.js b/extension/src/commands/highlight-detection.injected.js
index ad31ef5..72f3e8e 100644
--- a/extension/src/commands/highlight-detection.injected.js
+++ b/extension/src/commands/highlight-detection.injected.js
@@ -2498,10 +2498,7 @@ function collectHighlightCandidates(config, trace, layoutStability) {
       droppable: 0,
       uploadable: uploadableElements.length,
     };
-    trace(
-      'scan:done',
-      `mode=uploadable matched=${uploadableElements.length}`,
-    );
+    trace('scan:done', `mode=uploadable matched=${uploadableElements.length}`);
     return { elements: uploadableElements, counts: uploadableCounts };
   }
 
diff --git a/server/api/routes/agent.py b/server/api/routes/agent.py
index 213e72a..4d03f65 100644
--- a/server/api/routes/agent.py
+++ b/server/api/routes/agent.py
@@ -92,9 +92,11 @@ def _validate_images(
                 "data_uri": data_uri,
                 "mime_type": mime_type or entry.get("mime_type"),
                 "name": entry.get("name") or f"image-{i + 1}",
-                "size_bytes": entry.get("size_bytes")
-                if isinstance(entry.get("size_bytes"), int)
-                else decoded_size,
+                "size_bytes": (
+                    entry.get("size_bytes")
+                    if isinstance(entry.get("size_bytes"), int)
+                    else decoded_size
+                ),
             }
         )
     return normalized
diff --git a/server/core/browser_executor_bundle.py b/server/core/browser_executor_bundle.py
index 6837232..0f051c6 100644
--- a/server/core/browser_executor_bundle.py
+++ b/server/core/browser_executor_bundle.py
@@ -282,9 +282,7 @@ async def execute_agent_message(
             if images:
                 from openhands.sdk.llm import Message, TextContent, ImageContent
 
-                image_urls = [
-                    img["data_uri"] for img in images if img.get("data_uri")
-                ]
+                image_urls = [img["data_uri"] for img in images if img.get("data_uri")]
                 content: list[Any] = [TextContent(text=message_text)]
                 if image_urls:
                     content.append(ImageContent(image_urls=image_urls))
diff --git a/server/core/processor.py b/server/core/processor.py
index c15a9d7..1bb55b2 100644
--- a/server/core/processor.py
+++ b/server/core/processor.py
@@ -458,9 +458,7 @@ async def _execute_highlight_drop_preview(
         """Highlight inner elements of a drop container for drag-and-drop 2PC"""
         return await self._send_prepared_command(command)
 
-    async def _execute_upload_file(
-        self, command: UploadFileCommand
-    ) -> CommandResponse:
+    async def _execute_upload_file(self, command: UploadFileCommand) -> CommandResponse:
         """Attach a local file to an <input type=file> via CDP.
 
         Validates the path server-side before dispatching to the extension,

From a80f3de679930f0b106fb952c2108c686d437cf9 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Thu, 16 Apr 2026 21:43:34 +0800
Subject: [PATCH 04/12] skill(open-browser): document portable ~/.claude/skills
 invocation path

The skill is now symlinked into ~/.claude/skills/open-browser/ for
global use. Update every `python3 skill/claude/open-browser/...`
reference to `python3 ~/.claude/skills/open-browser/...` so the same
command works from any project's CWD (including inside the OpenBrowser
repo, where the symlink still resolves back here).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 skill/claude/open-browser/SKILL.md               | 16 +++++++++-------
 skill/claude/open-browser/references/setup.md    |  2 +-
 .../claude/open-browser/scripts/check_status.py  |  2 +-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/skill/claude/open-browser/SKILL.md b/skill/claude/open-browser/SKILL.md
index 1130574..3b7478c 100644
--- a/skill/claude/open-browser/SKILL.md
+++ b/skill/claude/open-browser/SKILL.md
@@ -37,7 +37,7 @@ Before sending a browser task, confirm all of the following:
 Run this first:
 
 ```bash
-python3 skill/claude/open-browser/scripts/check_status.py --chrome-uuid "$OPENBROWSER_CHROME_UUID"
+python3 ~/.claude/skills/open-browser/scripts/check_status.py --chrome-uuid "$OPENBROWSER_CHROME_UUID"
 ```
 
 If readiness fails, read [references/setup.md](references/setup.md) or
@@ -72,7 +72,7 @@ Code, because the SSE stream becomes part of your conversation context
 without any extra plumbing:
 
 ```bash
-python3 skill/claude/open-browser/scripts/send_task.py \
+python3 ~/.claude/skills/open-browser/scripts/send_task.py \
   "Open https://example.com and report the page title" \
   --chrome-uuid "$OPENBROWSER_CHROME_UUID"
 ```
@@ -115,7 +115,7 @@ encoded, and sent as data URIs — no upload endpoint or static server is
 required. Limit: 10 MB per image, up to 8 images per message.
 
 ```bash
-python3 skill/claude/open-browser/scripts/send_task.py \
+python3 ~/.claude/skills/open-browser/scripts/send_task.py \
   "Open the local dashboard and tell me which section looks different from this screenshot." \
   --image /tmp/reference.png \
   --chrome-uuid "$OPENBROWSER_CHROME_UUID"
@@ -141,7 +141,7 @@ keeps its prior screenshots and observations), reuse the conversation
 ID from the previous run:
 
 ```bash
-python3 skill/claude/open-browser/scripts/send_task.py \
+python3 ~/.claude/skills/open-browser/scripts/send_task.py \
   "Now click the 'Sign in' button you just identified" \
   --chrome-uuid "$OPENBROWSER_CHROME_UUID" \
   --conversation-id 1b32b26a-1a7e-4b6c-9599-139fc6b9c89b
@@ -153,14 +153,16 @@ report a value it already saw.
 
 ## Working Directory
 
-Run commands from the OpenBrowser repo root so the relative script
-paths resolve cleanly.
+The skill's scripts live at `~/.claude/skills/open-browser/` so they
+work from any project's current working directory. The OpenBrowser
+server itself must still be started from the repo root
+(`uv run local-chrome-server serve` in `~/git/OpenBrowser`).
 
 Use `--cwd` when the browser task should operate with context from
 another workspace:
 
 ```bash
-python3 skill/claude/open-browser/scripts/send_task.py \
+python3 ~/.claude/skills/open-browser/scripts/send_task.py \
   "Open the local app and verify the login flow" \
   --cwd /absolute/path/to/project \
   --chrome-uuid "$OPENBROWSER_CHROME_UUID"
diff --git a/skill/claude/open-browser/references/setup.md b/skill/claude/open-browser/references/setup.md
index 5abbc1c..477596e 100644
--- a/skill/claude/open-browser/references/setup.md
+++ b/skill/claude/open-browser/references/setup.md
@@ -45,7 +45,7 @@ drive the browser that registered it.
 ## Quick verification
 
 ```bash
-python3 skill/claude/open-browser/scripts/check_status.py --chrome-uuid "$OPENBROWSER_CHROME_UUID"
+python3 ~/.claude/skills/open-browser/scripts/check_status.py --chrome-uuid "$OPENBROWSER_CHROME_UUID"
 ```
 
 Expected outcome:
diff --git a/skill/claude/open-browser/scripts/check_status.py b/skill/claude/open-browser/scripts/check_status.py
index c218162..8752bf7 100644
--- a/skill/claude/open-browser/scripts/check_status.py
+++ b/skill/claude/open-browser/scripts/check_status.py
@@ -136,7 +136,7 @@ def main() -> int:
         print("Ready for browser automation.")
         return 0
 
-    print("Not ready. See skill/claude/open-browser/references/setup.md if needed.")
+    print("Not ready. See ~/.claude/skills/open-browser/references/setup.md if needed.")
     return 1
 
 

From ac7fa57da1c8ebbf58a18bfb871908a8aa090609 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Fri, 17 Apr 2026 13:08:10 +0800
Subject: [PATCH 05/12] chore: update openhands-sdk and openhands-tools to
 bd4cb296355c3d03dd411883e78527b1915fa8c4

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
---
 pyproject.toml | 4 ++--
 uv.lock        | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index dd933be..69ae578 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -76,5 +76,5 @@ override-dependencies = [
 ]
 
 [tool.uv.sources]
-openhands-sdk = { git = "https://github.com/softpudding/agent-sdk.git", subdirectory = "openhands-sdk", rev = "764fb87256d7bc20b3eccf82c8a4d241e6740d63" }
-openhands-tools = { git = "https://github.com/softpudding/agent-sdk.git", subdirectory = "openhands-tools", rev = "764fb87256d7bc20b3eccf82c8a4d241e6740d63" }
+openhands-sdk = { git = "https://github.com/softpudding/agent-sdk.git", subdirectory = "openhands-sdk", rev = "bd4cb296355c3d03dd411883e78527b1915fa8c4" }
+openhands-tools = { git = "https://github.com/softpudding/agent-sdk.git", subdirectory = "openhands-tools", rev = "bd4cb296355c3d03dd411883e78527b1915fa8c4" }
diff --git a/uv.lock b/uv.lock
index 418acbe..36f3fc5 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1678,8 +1678,8 @@ requires-dist = [
     { name = "litellm", git = "https://github.com/softpudding/litellm.git?rev=2eb7db59461e9117b1e3e0519616b39f1497c0f9" },
     { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.7.0" },
     { name = "numpy", specifier = ">=1.24.0" },
-    { name = "openhands-sdk", git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-sdk&rev=764fb87256d7bc20b3eccf82c8a4d241e6740d63" },
-    { name = "openhands-tools", git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-tools&rev=764fb87256d7bc20b3eccf82c8a4d241e6740d63" },
+    { name = "openhands-sdk", git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-sdk&rev=bd4cb296355c3d03dd411883e78527b1915fa8c4" },
+    { name = "openhands-tools", git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-tools&rev=bd4cb296355c3d03dd411883e78527b1915fa8c4" },
     { name = "pillow", specifier = ">=10.0.0" },
     { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.0.0" },
     { name = "pydantic", specifier = ">=2.5.0" },
@@ -2224,7 +2224,7 @@ wheels = [
 [[package]]
 name = "openhands-sdk"
 version = "1.12.0"
-source = { git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-sdk&rev=764fb87256d7bc20b3eccf82c8a4d241e6740d63#764fb87256d7bc20b3eccf82c8a4d241e6740d63" }
+source = { git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-sdk&rev=bd4cb296355c3d03dd411883e78527b1915fa8c4#bd4cb296355c3d03dd411883e78527b1915fa8c4" }
 dependencies = [
     { name = "agent-client-protocol" },
     { name = "deprecation" },
@@ -2244,7 +2244,7 @@ dependencies = [
 [[package]]
 name = "openhands-tools"
 version = "1.12.0"
-source = { git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-tools&rev=764fb87256d7bc20b3eccf82c8a4d241e6740d63#764fb87256d7bc20b3eccf82c8a4d241e6740d63" }
+source = { git = "https://github.com/softpudding/agent-sdk.git?subdirectory=openhands-tools&rev=bd4cb296355c3d03dd411883e78527b1915fa8c4#bd4cb296355c3d03dd411883e78527b1915fa8c4" }
 dependencies = [
     { name = "bashlex" },
     { name = "binaryornot" },

From f046eed0d9455e5644fa5f465ba4be963b721cbb Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Fri, 17 Apr 2026 13:17:32 +0800
Subject: [PATCH 06/12] =?UTF-8?q?skill(ob-routines):=20add=20Browser=20Rou?=
 =?UTF-8?q?tines=20skill=20(record=20=E2=86=92=20compile=20=E2=86=92=20rep?=
 =?UTF-8?q?lay)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Introduces the ob-routines skill (alias for openbrowser-routines) for
capturing, compiling, and replaying named Chrome workflows. Previously lived
only in ~/.claude/skills/routines/; now versioned under skill/claude/ob-routines/
so it can be installed via symlink alongside open-browser.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 skill/claude/ob-routines/SKILL.md             | 235 +++++++++++
 skill/claude/ob-routines/scripts/compile.py   | 390 ++++++++++++++++++
 .../ob-routines/scripts/list_routines.py      | 121 ++++++
 skill/claude/ob-routines/scripts/replay.py    | 346 ++++++++++++++++
 .../ob-routines/scripts/start_recording.py    | 123 ++++++
 .../ob-routines/scripts/stop_recording.py     |  90 ++++
 6 files changed, 1305 insertions(+)
 create mode 100644 skill/claude/ob-routines/SKILL.md
 create mode 100644 skill/claude/ob-routines/scripts/compile.py
 create mode 100644 skill/claude/ob-routines/scripts/list_routines.py
 create mode 100644 skill/claude/ob-routines/scripts/replay.py
 create mode 100644 skill/claude/ob-routines/scripts/start_recording.py
 create mode 100644 skill/claude/ob-routines/scripts/stop_recording.py

diff --git a/skill/claude/ob-routines/SKILL.md b/skill/claude/ob-routines/SKILL.md
new file mode 100644
index 0000000..d9ecb7a
--- /dev/null
+++ b/skill/claude/ob-routines/SKILL.md
@@ -0,0 +1,235 @@
+---
+name: ob-routines
+description: Record, compile, and replay Browser Routines — saved, named browser workflows. (Alias for openbrowser-routines.) Supports subcommands: "list [query]" to list/search routines, "new" to record a new routine, "execute <name>" to replay a saved routine. Use when the user says "list routines", "record a routine", "replay X", "execute X", or "/ob-routines <subcommand>".
+---
+
+# Browser Routines
+
+Browser Routines are named, compiled workflows captured from real Chrome sessions.
+The pipeline has four stages: **record → compile → name → replay**.
+
+## Subcommand dispatch
+
+When invoked with arguments, act immediately — do not ask the user what they want:
+
+| Invocation | Action |
+|---|---|
+| `/ob-routines` | Show available routines and ask what to do |
+| `/ob-routines list [query]` | Run `list_routines.py [query]` and display results |
+| `/ob-routines new` | Ask what flow to record, then start the full record → compile pipeline |
+| `/ob-routines execute <name>` | Run `replay.py <name>` immediately |
+
+---
+
+## Your role during compilation
+
+You are a **bridge and quality gate**, not the compiler. The Compiler Agent does
+the reasoning; you ensure it did its job correctly before finalizing.
+
+### Bridge duties
+1. Run `compile.py` in a tmux pane (mandatory — see below).
+2. Watch for `[compiler:question]` — relay it to the user, send their answer back.
+3. Watch for `[compiler:stalled]` — show the agent's message, optionally prompt a follow-up.
+4. At `[compiler:name_prompt]` — help the user pick a short slug.
+
+### Quality gate (run before every finalize)
+
+After the compiler reports `status=review`, read the compiled routine markdown
+and check **both** of the following before calling `/compile/finalize`:
+
+#### Gate 1 — Intent clarity
+Did the compiler understand *why* the user performed each action, not just *what*
+they clicked? Red flags:
+- Steps that say "click X" with no explanation of goal or condition
+- A position-based selection from a sorted/filtered list without asking whether
+  to replay by position or by identity (e.g. "upvote the top 3 posts" — top 3
+  today vs. the same 3 posts always?)
+- A value (date, search query, ticker, ID) that will obviously change between
+  runs, not parameterized
+
+If any red flag is present and the compiler did NOT ask about it: relay the
+ambiguity to the user yourself, get their answer, then send it via
+`POST /recordings/{id}/compile/answer` so the compiler can revise.
+
+#### Gate 2 — Delivery goal for read-only workflows
+
+A workflow is **read-only** if it has no form submission, no purchase, no
+send/post/create/delete action — the user only navigated, read, filtered, or
+inspected. For read-only workflows, ask: does the compiled routine end with a
+delivery step (a `file_editor` write, a `terminal` command, or an explicit
+instruction to report results in chat)?
+
+**If the routine is read-only AND has no delivery step, the compiler made an
+error.** Do not finalize. Instead:
+
+1. Tell the user: "This routine reads data but doesn't capture results anywhere.
+   How do you want results delivered on replay?"
+   - (a) Summary shown in chat (brief / structured table / full details?)
+   - (b) Written to a local file (path + format: plain text, Markdown, CSV, JSON?)
+   - (c) Both
+2. Get their answer.
+3. Send it to the compiler via `POST /recordings/{id}/compile/answer` — the
+   compiler will revise the routine to include the delivery step.
+4. Wait for the next `status=review`, then re-run both gates.
+
+> **Why this matters:** A routine that just clicks through pages is useless on
+> replay — OpenBrowser will navigate and stop with no output. The delivery step
+> is what makes the routine meaningful.
+
+---
+
+## Preconditions
+
+**First time?** Complete the full setup in `skill/claude/open-browser/references/setup.md`
+before using this skill. That guide covers: loading the Chrome extension, connecting
+it to the server, and obtaining a valid `OPENBROWSER_CHROME_UUID`. Without that,
+recording and replay will fail immediately.
+
+For subsequent uses, confirm:
+- OpenBrowser server at `http://127.0.0.1:8765`
+- Chrome extension connected
+- `OPENBROWSER_CHROME_UUID` set (or passed via `--chrome-uuid`)
+
+Quick check:
+```bash
+python3 skill/claude/open-browser/scripts/check_status.py --chrome-uuid "$OPENBROWSER_CHROME_UUID"
+```
+
+Start the server if needed:
+```bash
+cd /Users/yangxiao/git/OpenBrowser && uv run local-chrome-server serve
+```
+
+Scripts path: `skill/claude/ob-routines/scripts/` (run from repo root).
+
+---
+
+## List & search routines
+
+```bash
+python3 skill/claude/ob-routines/scripts/list_routines.py
+python3 skill/claude/ob-routines/scripts/list_routines.py "login"
+python3 skill/claude/ob-routines/scripts/list_routines.py --recordings
+```
+
+---
+
+## Record a routine
+
+### Step 1 — start recording
+```bash
+python3 skill/claude/ob-routines/scripts/start_recording.py \
+  --chrome-uuid "$OPENBROWSER_CHROME_UUID" \
+  --name "xiaohongshu-messages" \
+  --intent "check messages on Xiaohongshu"
+```
+
+Prints `[recording:started] <recording_id>`. **Save this ID.**
+
+Tell the user: **"Perform your actions in the browser window, then come back and say done."**
+Do NOT proceed until the user confirms.
+
+### Step 2 — stop recording
+```bash
+python3 skill/claude/ob-routines/scripts/stop_recording.py <recording_id>
+```
+
+---
+
+## Compile to a routine — MANDATORY: tmux interactive session
+
+**compile.py uses `input()` for Q&A and the name prompt. It MUST run in an
+interactive shell. Never invoke it directly via the Bash tool — it will block
+and then be killed, losing the compiler session.**
+
+### Launch in tmux
+```bash
+tmux new-window -n "compile" \
+  "cd /Users/yangxiao/git/OpenBrowser && python3 skill/claude/ob-routines/scripts/compile.py <recording_id>; echo '[compile-done]'"
+```
+
+### Monitor output
+```bash
+tmux capture-pane -t "compile" -p
+```
+
+### Send an answer
+```bash
+tmux send-keys -t "compile" "the answer" Enter
+```
+
+### Markers to watch for
+
+| Marker | Your action |
+|---|---|
+| `[compiler:thought]` / `[compiler:action]` | Relay as progress to user |
+| `[compiler:question] <text>` | Relay to user, wait for answer, send via `tmux send-keys` |
+| `[compiler:stalled] <text>` | Show message, ask user for follow-up |
+| `[compiler:complete] goal=… steps=N` | Compilation reached review state |
+| `[compiler:routine_draft]` | Full routine markdown printed for inspection |
+| `[compiler:gate_check]` | **Run both quality gates here.** Send feedback or press Enter |
+| `[compiler:name_prompt]` | Gates passed — help user pick slug |
+| `[compiler:saved]` | Done — report name and id |
+
+### Quality gate checkpoint
+When `[compiler:gate_check]` appears in the pane, compile.py is explicitly
+paused waiting for your review of `[compiler:routine_draft]`. Run Gate 1 and Gate 2:
+
+- **Gates pass** → send an empty Enter: `tmux send-keys -t main:compile "" Enter`
+- **Gate fails** → send corrective feedback:
+  `tmux send-keys -t main:compile "Please add a delivery step: summarise results in chat as a structured list of tickers with metrics." Enter`
+
+compile.py forwards non-empty input back to the compiler, streams the revision,
+and loops back to another `[compiler:gate_check]`. Only an empty Enter advances
+to `[compiler:name_prompt]`.
+
+**Never send gate feedback at the `[compiler:name_prompt]` stage** — that input
+goes directly to the routine name field, not the compiler.
+
+---
+
+## Replay a routine
+
+```bash
+python3 skill/claude/ob-routines/scripts/replay.py "routine-name" \
+  --chrome-uuid "$OPENBROWSER_CHROME_UUID"
+
+# List without replaying
+python3 skill/claude/ob-routines/scripts/replay.py --list
+```
+
+Name matching: exact → ID → prefix → substring.
+
+---
+
+## Full example workflow
+
+```
+1. /ob-routines new  →  ask user what to record
+2. start_recording  →  [recording:started] abc123
+3. (user records in browser, says "done")
+4. stop_recording abc123  →  [recording:events] 21 events
+5. tmux new-window "compile.py abc123"
+6. monitor pane → relay questions → send answers
+7. [compiler:complete]  →  run Gate 1 + Gate 2
+   Gate 2 fails: routine is read-only, no delivery step
+   → ask user: chat summary, file, or both?
+   → send answer via tmux send-keys
+   → wait for next [compiler:complete]
+8. Gates pass → [compiler:name_prompt] → user picks slug
+9. [compiler:saved] name='…' id=…
+10. /ob-routines execute <name>  →  streams [action] … [complete]
+```
+
+---
+
+## Failure handling
+
+- **Server unreachable**: `uv run local-chrome-server serve`
+- **Browser UUID invalid**: reconnect Chrome extension, get fresh UUID
+- **0 events captured**: browser disconnected; re-record
+- **tmux not found**: `brew install tmux`
+- **tmux window conflict**: check `tmux list-windows`, use a unique `-n` name
+- **Compiler session expired** (pane exited before finalize): call
+  `POST /recordings/{id}/compile` again to restart — session is fresh
+- **Relay stuck**: `[observation:error]` lines in SSE stream; relay to user
diff --git a/skill/claude/ob-routines/scripts/compile.py b/skill/claude/ob-routines/scripts/compile.py
new file mode 100644
index 0000000..5ed9a92
--- /dev/null
+++ b/skill/claude/ob-routines/scripts/compile.py
@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+"""Compile a stopped recording into a named Browser Routine.
+
+Starts the Compiler Agent, streams its SSE output, and acts as a bridge
+between the agent and the user:
+  - Agent reasoning and tool calls are printed to stdout as they arrive.
+  - When the compiler agent asks a clarification question (status=asking),
+    this script prints the question and reads the user's answer from stdin,
+    then resumes compilation via /compile/answer.
+  - When the agent stalls (status=stalled), the agent's last message is
+    shown and the user can send a follow-up.
+  - When compilation completes (status=review), the script prompts the user
+    to name the routine, then calls /compile/finalize to save it.
+
+The outer agent (Claude Code / Codex) should relay the printed questions to
+the user and feed their responses back via stdin — it should NOT try to
+re-implement compiler logic.
+
+Example:
+  python3 compile.py abc123-recording-id
+  python3 compile.py abc123-recording-id --model-alias fast
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+
+# ---------------------------------------------------------------------------
+# HTTP helpers
+# ---------------------------------------------------------------------------
+
+
+def request_json(
+    url: str,
+    *,
+    method: str = "GET",
+    body: dict | None = None,
+    timeout: int = 15,
+) -> dict:
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    data = None if body is None else json.dumps(body).encode("utf-8")
+    req = Request(url, data=data, headers=headers, method=method)
+    with urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read().decode("utf-8"))
+
+
+# ---------------------------------------------------------------------------
+# SSE event formatting  (same conventions as send_task.py)
+# ---------------------------------------------------------------------------
+
+
+def _format_compiler_event(event_type: str, data: dict) -> None:
+    """Print one SSE event from the compiler agent stream."""
+    if event_type == "error":
+        print(f"[compiler:error] {data.get('error', data)}", flush=True)
+        return
+
+    if event_type != "agent_event":
+        # Pass-through for unknown top-level event types
+        print(f"[{event_type}] {json.dumps(data, ensure_ascii=False)}", flush=True)
+        return
+
+    data_type = data.get("type", "unknown")
+
+    if data_type == "SystemPromptEvent":
+        text_len = len(data.get("text", ""))
+        print(
+            f"[compiler:system_prompt] suppressed ({text_len} chars)",
+            flush=True,
+        )
+        return
+
+    if data_type == "ThoughtEvent":
+        thought = data.get("thought", data.get("content", ""))
+        print(f"[compiler:thought] {thought}", flush=True)
+        return
+
+    if data_type == "ActionEvent":
+        action = data.get("action", {})
+        if isinstance(action, dict):
+            action_name = action.get("action", "unknown")
+            if action_name == "ask_user":
+                question = action.get("question", "")
+                print(f"[compiler:ask_user] {question}", flush=True)
+            else:
+                # FileEditorTool, TraceViewerTool, SubmitWorkflowTool, etc.
+                extras = {
+                    k: v for k, v in action.items()
+                    if k != "action" and v is not None
+                }
+                suffix = (" " + json.dumps(extras, ensure_ascii=False)) if extras else ""
+                print(f"[compiler:action] {action_name}{suffix}", flush=True)
+        else:
+            print(f"[compiler:action] {action}", flush=True)
+        return
+
+    if data_type == "ObservationEvent":
+        success = data.get("success", False)
+        message = data.get("message", "")
+        state = "ok" if success else "error"
+        print(f"[compiler:observation:{state}] {message}", flush=True)
+        return
+
+    if data_type == "MessageEvent":
+        role = data.get("role", "unknown")
+        text = data.get("text", "")
+        print(f"[compiler:message:{role}] {text}", flush=True)
+        return
+
+    if data_type == "ErrorEvent":
+        print(f"[compiler:error] {data.get('error', 'unknown error')}", flush=True)
+        return
+
+    print(
+        f"[compiler:agent_event:{data_type}] {json.dumps(data, ensure_ascii=False)}",
+        flush=True,
+    )
+
+
+# ---------------------------------------------------------------------------
+# SSE streaming
+# ---------------------------------------------------------------------------
+
+
+def _stream_sse(url: str, body: dict) -> dict | None:
+    """POST to url with body, stream SSE events, return the final complete result.
+
+    Returns the ``result`` dict from the complete event, or None on error.
+    """
+    req = Request(
+        url,
+        data=json.dumps(body).encode("utf-8"),
+        headers={
+            "Content-Type": "application/json",
+            "Accept": "text/event-stream",
+        },
+        method="POST",
+    )
+
+    complete_result: dict | None = None
+    sse_event: str | None = None
+    sse_data: str | None = None
+
+    try:
+        with urlopen(req, timeout=None) as response:
+            for raw_line in response:
+                line = raw_line.decode("utf-8").rstrip("\n")
+                if not line:
+                    if sse_event and sse_data is not None:
+                        try:
+                            parsed = json.loads(sse_data)
+                        except json.JSONDecodeError:
+                            parsed = {"raw": sse_data}
+
+                        if sse_event == "complete":
+                            complete_result = parsed.get("result", parsed)
+                        else:
+                            _format_compiler_event(sse_event, parsed)
+
+                    sse_event = None
+                    sse_data = None
+                    continue
+
+                if line.startswith("event:"):
+                    sse_event = line[6:].strip()
+                elif line.startswith("data:"):
+                    sse_data = line[5:].lstrip()
+
+    except HTTPError as exc:
+        body_text = exc.read().decode("utf-8", errors="replace")
+        print(f"[compiler:http_error] {exc.code} {exc.reason}: {body_text}", file=sys.stderr)
+        return None
+
+    return complete_result
+
+
+# ---------------------------------------------------------------------------
+# Compile loop
+# ---------------------------------------------------------------------------
+
+
+def compile_recording(base_url: str, recording_id: str, model_alias: str | None) -> int:
+    """Run the compile → Q&A → finalize flow. Returns exit code."""
+    print(f"[compiler:start] recording={recording_id}", flush=True)
+
+    # ── Phase 1: initial compile ──────────────────────────────────────────
+    compile_body: dict = {}
+    if model_alias:
+        compile_body["model_alias"] = model_alias
+
+    result = _stream_sse(
+        f"{base_url}/recordings/{recording_id}/compile",
+        body=compile_body,
+    )
+    if result is None:
+        return 1
+
+    # ── Phase 2: Q&A loop ─────────────────────────────────────────────────
+    while True:
+        status = result.get("status")
+
+        if status == "asking":
+            question = result.get("question", "")
+            print(f"\n[compiler:question] {question}", flush=True)
+            print("[compiler:waiting_for_answer] Type your answer and press Enter:",
+                  flush=True)
+            try:
+                answer = input().strip()
+            except (EOFError, KeyboardInterrupt):
+                print("\n[compiler:interrupted] Compilation cancelled.", flush=True)
+                return 130
+
+            result = _stream_sse(
+                f"{base_url}/recordings/{recording_id}/compile/answer",
+                body={"answer": answer},
+            )
+            if result is None:
+                return 1
+
+        elif status == "stalled":
+            # Agent replied in prose instead of calling ask_user.
+            # Show the message and let the user send a follow-up.
+            message = result.get("message", "")
+            if message:
+                print(f"\n[compiler:stalled] {message}", flush=True)
+            print(
+                "[compiler:waiting_for_follow_up] Agent stalled — send a follow-up "
+                "(or press Enter to continue without one):",
+                flush=True,
+            )
+            try:
+                follow_up = input().strip()
+            except (EOFError, KeyboardInterrupt):
+                print("\n[compiler:interrupted] Compilation cancelled.", flush=True)
+                return 130
+
+            if not follow_up:
+                follow_up = "Please continue."
+
+            result = _stream_sse(
+                f"{base_url}/recordings/{recording_id}/compile/answer",
+                body={"answer": follow_up},
+            )
+            if result is None:
+                return 1
+
+        elif status == "review":
+            # Compilation done — show the draft and pause for quality gate
+            # before proceeding to the name prompt. The outer agent (Claude
+            # Code / Codex) reads the routine here and may send corrective
+            # feedback (e.g. missing delivery step) via the gate prompt.
+            # Only an empty Enter moves forward to naming.
+            goal = result.get("goal", "")
+            step_count = result.get("step_count", "?")
+            routine_markdown = result.get("routine_markdown", "")
+            print(f"\n[compiler:complete] goal={goal!r}  steps={step_count}", flush=True)
+            if routine_markdown:
+                print(f"[compiler:routine_draft]\n{routine_markdown}", flush=True)
+            print(
+                "\n[compiler:gate_check] Review the routine above.\n"
+                "Press Enter to proceed to naming, or type feedback to send back to the compiler:",
+                flush=True,
+            )
+            try:
+                gate_input = input().strip()
+            except (EOFError, KeyboardInterrupt):
+                print("\n[compiler:interrupted] Compilation cancelled.", flush=True)
+                return 130
+
+            if gate_input:
+                # Outer agent has feedback — send it back to the compiler
+                result = _stream_sse(
+                    f"{base_url}/recordings/{recording_id}/compile/answer",
+                    body={"answer": gate_input},
+                )
+                if result is None:
+                    return 1
+                # Loop back to handle the next status
+                continue
+
+            # Gate passed — proceed to naming
+            break
+
+        else:
+            print(
+                f"[compiler:unexpected_status] {status} — result: {result}",
+                file=sys.stderr,
+            )
+            return 1
+
+    # ── Phase 3: name the routine and finalize ────────────────────────────
+    goal = result.get("goal", "")
+    step_count = result.get("step_count", "?")
+
+    # Suggest a slug derived from the goal
+    suggested = _slugify(goal) if goal else "my-routine"
+    print(
+        f"\n[compiler:name_prompt] Suggested name: {suggested!r}\n"
+        f"Accept (press Enter) or type a new name:",
+        flush=True,
+    )
+    try:
+        chosen_name = input().strip()
+    except (EOFError, KeyboardInterrupt):
+        print("\n[compiler:interrupted] Finalization cancelled.", flush=True)
+        return 130
+
+    if not chosen_name:
+        chosen_name = suggested
+
+    # ── Phase 4: finalize ─────────────────────────────────────────────────
+    try:
+        finalize_result = request_json(
+            f"{base_url}/recordings/{recording_id}/compile/finalize",
+            method="POST",
+            body={"name": chosen_name},
+        )
+    except HTTPError as exc:
+        body_text = exc.read().decode("utf-8", errors="replace")
+        print(f"[compiler:finalize_error] {exc.code}: {body_text}", file=sys.stderr)
+        return 1
+    except Exception as exc:
+        print(f"[compiler:finalize_error] {exc}", file=sys.stderr)
+        return 1
+
+    routine = finalize_result.get("routine", {})
+    routine_id = routine.get("routine_id", "?")
+    name = routine.get("name", chosen_name)
+    steps = routine.get("step_count", "?")
+
+    print(f"[compiler:saved] name={name!r}  id={routine_id}  steps={steps}", flush=True)
+    print(
+        f"\nRoutine saved. To replay it, run:\n\n"
+        f"  python3 replay.py {name!r}\n",
+        flush=True,
+    )
+    return 0
+
+
+def _slugify(text: str) -> str:
+    """Turn a goal string into a short, lowercase, hyphenated slug."""
+    import re
+    # Lowercase, keep only alnum and spaces, collapse and replace with hyphens
+    slug = re.sub(r"[^\w\s]", "", text.lower())
+    slug = re.sub(r"\s+", "-", slug.strip())
+    # Truncate to 40 chars, trim trailing hyphens
+    slug = slug[:40].rstrip("-")
+    return slug or "routine"
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Compile a stopped recording into a named Browser Routine",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument("recording_id", help="Recording ID from stop_recording.py")
+    parser.add_argument(
+        "--model-alias",
+        help="LLM model alias to use for compilation (uses server default if omitted)",
+    )
+    parser.add_argument(
+        "--url",
+        default="http://127.0.0.1:8765",
+        help="OpenBrowser server URL",
+    )
+    args = parser.parse_args()
+
+    try:
+        return compile_recording(args.url, args.recording_id, args.model_alias)
+    except URLError as exc:
+        print(f"Cannot reach OpenBrowser server: {exc}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        print("Interrupted.", file=sys.stderr)
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/skill/claude/ob-routines/scripts/list_routines.py b/skill/claude/ob-routines/scripts/list_routines.py
new file mode 100644
index 0000000..24a1dac
--- /dev/null
+++ b/skill/claude/ob-routines/scripts/list_routines.py
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""List saved routines and/or stopped recordings.
+
+Routines are named, compiled browser workflows ready to replay.
+Recordings are raw captured traces that may not yet be compiled.
+
+Examples:
+  python3 list_routines.py                    # list all routines
+  python3 list_routines.py login              # filter by name/goal substring
+  python3 list_routines.py --recordings       # list stopped recordings instead
+  python3 list_routines.py --recordings login # filter recordings by name
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from urllib.error import URLError
+from urllib.request import Request, urlopen
+
+
+def request_json(url: str, *, timeout: int = 10) -> dict:
+    req = Request(url, headers={"Accept": "application/json"})
+    with urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read().decode("utf-8"))
+
+
+def list_routines(base_url: str, query: str | None) -> int:
+    try:
+        data = request_json(f"{base_url}/routines")
+    except URLError as exc:
+        print(f"Cannot reach OpenBrowser server: {exc}", file=sys.stderr)
+        return 1
+
+    items = data.get("routines", [])
+    if query:
+        q = query.lower()
+        items = [
+            r for r in items
+            if q in r["name"].lower() or q in r.get("goal", "").lower()
+        ]
+
+    if not items:
+        suffix = f" matching {query!r}" if query else ""
+        print(f"No routines found{suffix}.")
+        return 0
+
+    print(f"{'NAME':<30}  {'STEPS':>5}  {'GOAL'}")
+    print("-" * 72)
+    for r in items:
+        name = r["name"]
+        steps = r.get("step_count", "?")
+        goal = r.get("goal", "")
+        routine_id = r["routine_id"]
+        print(f"{name:<30}  {steps:>5}  {goal}")
+        print(f"  id={routine_id}")
+    return 0
+
+
+def list_recordings(base_url: str, query: str | None) -> int:
+    try:
+        data = request_json(f"{base_url}/recordings?status=stopped")
+    except URLError as exc:
+        print(f"Cannot reach OpenBrowser server: {exc}", file=sys.stderr)
+        return 1
+
+    items = data.get("recordings", [])
+    if query:
+        q = query.lower()
+        items = [
+            r for r in items
+            if q in (r.get("name") or "").lower()
+        ]
+
+    if not items:
+        suffix = f" matching {query!r}" if query else ""
+        print(f"No stopped recordings found{suffix}.")
+        return 0
+
+    print(f"{'NAME':<30}  {'EVENTS':>6}  {'RECORDING ID'}")
+    print("-" * 72)
+    for r in items:
+        name = r.get("name") or "(unnamed)"
+        events = r.get("event_count", "?")
+        recording_id = r["recording_id"]
+        compiled = "(compiled)" if (r.get("metadata") or {}).get("routine_id") else ""
+        print(f"{name:<30}  {events:>6}  {recording_id}  {compiled}")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="List saved routines or stopped recordings",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument(
+        "query",
+        nargs="?",
+        help="Filter by name or goal substring (case-insensitive)",
+    )
+    parser.add_argument(
+        "--recordings",
+        action="store_true",
+        help="List stopped recordings instead of compiled routines",
+    )
+    parser.add_argument(
+        "--url",
+        default="http://127.0.0.1:8765",
+        help="OpenBrowser server URL",
+    )
+    args = parser.parse_args()
+
+    if args.recordings:
+        return list_recordings(args.url, args.query)
+    return list_routines(args.url, args.query)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/skill/claude/ob-routines/scripts/replay.py b/skill/claude/ob-routines/scripts/replay.py
new file mode 100644
index 0000000..5520fb4
--- /dev/null
+++ b/skill/claude/ob-routines/scripts/replay.py
@@ -0,0 +1,346 @@
+#!/usr/bin/env python3
+"""Execute a saved Browser Routine in Chrome.
+
+Looks up the routine by name (exact or prefix match, case-insensitive),
+creates an agent conversation in routine_replay mode, sends the routine
+markdown as the task, and streams execution output.
+
+Examples:
+  python3 replay.py "techforum-upvote" --chrome-uuid "$OPENBROWSER_CHROME_UUID"
+  python3 replay.py login               # prefix match
+  python3 replay.py --list              # list all available routines
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from urllib.error import URLError
+from urllib.request import Request, urlopen
+
+
+# ---------------------------------------------------------------------------
+# HTTP helpers
+# ---------------------------------------------------------------------------
+
+
+def request_json(
+    url: str,
+    *,
+    method: str = "GET",
+    body: dict | None = None,
+    timeout: int = 10,
+) -> dict:
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    data = None if body is None else json.dumps(body).encode("utf-8")
+    req = Request(url, data=data, headers=headers, method=method)
+    with urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read().decode("utf-8"))
+
+
+# ---------------------------------------------------------------------------
+# Routine lookup
+# ---------------------------------------------------------------------------
+
+
+def find_routine(base_url: str, query: str) -> dict | None:
+    """Return a single routine matching query by exact name, then prefix, then substring."""
+    data = request_json(f"{base_url}/routines")
+    routines = data.get("routines", [])
+    if not routines:
+        return None
+
+    q = query.lower()
+
+    # 1. Exact name match
+    for r in routines:
+        if r["name"].lower() == q:
+            return r
+
+    # 2. Exact routine_id match
+    for r in routines:
+        if r["routine_id"].lower() == q:
+            return r
+
+    # 3. Prefix match on name
+    prefix = [r for r in routines if r["name"].lower().startswith(q)]
+    if len(prefix) == 1:
+        return prefix[0]
+    if len(prefix) > 1:
+        print("[replay:ambiguous] Multiple routines match that prefix:", flush=True)
+        for r in prefix:
+            print(f"  {r['name']}  (id={r['routine_id']})", flush=True)
+        print("Provide a more specific name or the full routine_id.", flush=True)
+        return None
+
+    # 4. Substring match on name or goal
+    sub = [
+        r for r in routines
+        if q in r["name"].lower() or q in r.get("goal", "").lower()
+    ]
+    if len(sub) == 1:
+        return sub[0]
+    if len(sub) > 1:
+        print("[replay:ambiguous] Multiple routines match that substring:", flush=True)
+        for r in sub:
+            print(f"  {r['name']}  (id={r['routine_id']})", flush=True)
+        print("Provide a more specific name or the full routine_id.", flush=True)
+        return None
+
+    return None
+
+
+# ---------------------------------------------------------------------------
+# SSE streaming  (same conventions as send_task.py)
+# ---------------------------------------------------------------------------
+
+
+def _format_event(event_type: str, data: dict) -> None:
+    if event_type == "complete":
+        print(f"[complete] {data.get('message', '')}", flush=True)
+        return
+
+    if event_type == "usage_metrics":
+        metrics = data.get("metrics", {})
+        model_name = metrics.get("model_name", "unknown")
+        cost = metrics.get("accumulated_cost", 0)
+        token_usage = metrics.get("accumulated_token_usage", {})
+        total_tokens = token_usage.get("total_tokens", 0)
+        if total_tokens == 0:
+            total_tokens = (
+                token_usage.get("prompt_tokens", 0)
+                + token_usage.get("completion_tokens", 0)
+                + token_usage.get("reasoning_tokens", 0)
+            )
+        print(
+            f"[usage] model={model_name} cost_rmb={cost:.6f} tokens={total_tokens}",
+            flush=True,
+        )
+        return
+
+    if event_type != "agent_event":
+        print(f"[{event_type}] {json.dumps(data, ensure_ascii=False)}", flush=True)
+        return
+
+    data_type = data.get("type", "unknown")
+
+    if data_type == "SystemPromptEvent":
+        text_len = len(data.get("text", ""))
+        print(
+            f"[system_prompt] suppressed ({text_len} chars)",
+            flush=True,
+        )
+        return
+
+    if data_type == "MessageEvent":
+        role = data.get("role", "unknown")
+        text = data.get("text", "")
+        print(f"[message:{role}] {text}", flush=True)
+        return
+
+    if data_type == "ThoughtEvent":
+        thought = data.get("thought", data.get("content", ""))
+        print(f"[thought] {thought}", flush=True)
+        return
+
+    if data_type == "ActionEvent":
+        action = data.get("action", {})
+        if isinstance(action, dict):
+            action_name = action.get("action", "unknown")
+            element_id = action.get("element_id")
+            url = action.get("url")
+            text = action.get("text")
+            extras = []
+            if element_id:
+                extras.append(f"element_id={element_id}")
+            if url:
+                extras.append(f"url={url}")
+            if text:
+                extras.append(f"text={text!r}")
+            suffix = (" " + " ".join(extras)) if extras else ""
+            print(f"[action] {action_name}{suffix}", flush=True)
+        else:
+            print(f"[action] {action}", flush=True)
+        return
+
+    if data_type == "ObservationEvent":
+        success = data.get("success", False)
+        message = data.get("message", "")
+        state = "ok" if success else "error"
+        print(f"[observation:{state}] {message}", flush=True)
+        return
+
+    if data_type == "ErrorEvent":
+        print(f"[error] {data.get('error', 'unknown error')}", flush=True)
+        return
+
+    print(
+        f"[agent_event:{data_type}] {json.dumps(data, ensure_ascii=False)}",
+        flush=True,
+    )
+
+
+def stream_replay(
+    base_url: str,
+    conversation_id: str,
+    task: str,
+    cwd: str,
+    chrome_uuid: str,
+) -> None:
+    req = Request(
+        f"{base_url}/agent/conversations/{conversation_id}/messages",
+        data=json.dumps({
+            "text": task,
+            "cwd": cwd,
+            "browser_id": chrome_uuid,
+        }).encode("utf-8"),
+        headers={
+            "Content-Type": "application/json",
+            "Accept": "text/event-stream",
+        },
+        method="POST",
+    )
+
+    with urlopen(req, timeout=None) as response:
+        sse_event: str | None = None
+        sse_data: str | None = None
+        for raw_line in response:
+            line = raw_line.decode("utf-8").rstrip("\n")
+            if not line:
+                if sse_event and sse_data is not None:
+                    try:
+                        _format_event(sse_event, json.loads(sse_data))
+                    except json.JSONDecodeError:
+                        print(f"[{sse_event}] {sse_data}", flush=True)
+                sse_event = None
+                sse_data = None
+                continue
+
+            if line.startswith("event:"):
+                sse_event = line[6:].strip()
+            elif line.startswith("data:"):
+                sse_data = line[5:].lstrip()
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Replay a saved Browser Routine in Chrome",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument(
+        "routine",
+        nargs="?",
+        help="Routine name, ID, or prefix to replay",
+    )
+    parser.add_argument(
+        "--chrome-uuid",
+        default=os.environ.get("OPENBROWSER_CHROME_UUID"),
+        help="Browser UUID capability token (or set OPENBROWSER_CHROME_UUID)",
+    )
+    parser.add_argument(
+        "--cwd",
+        default=".",
+        help="Working directory passed to the agent",
+    )
+    parser.add_argument(
+        "--list",
+        action="store_true",
+        help="List available routines and exit",
+    )
+    parser.add_argument(
+        "--url",
+        default="http://127.0.0.1:8765",
+        help="OpenBrowser server URL",
+    )
+    args = parser.parse_args()
+
+    try:
+        if args.list or not args.routine:
+            data = request_json(f"{args.url}/routines")
+            routines = data.get("routines", [])
+            if not routines:
+                print("No routines saved yet.")
+                return 0
+            print(f"{'NAME':<30}  {'STEPS':>5}  GOAL")
+            print("-" * 72)
+            for r in routines:
+                print(f"{r['name']:<30}  {r.get('step_count', '?'):>5}  {r.get('goal', '')}")
+            return 0
+
+        if not args.chrome_uuid:
+            print(
+                "Browser UUID is required. Set OPENBROWSER_CHROME_UUID or pass --chrome-uuid.",
+                file=sys.stderr,
+            )
+            return 2
+
+        # ── Find the routine ──────────────────────────────────────────────
+        routine = find_routine(args.url, args.routine)
+        if routine is None:
+            print(
+                f"[replay:not_found] No routine found matching {args.routine!r}. "
+                "Run with --list to see available routines.",
+                file=sys.stderr,
+            )
+            return 1
+
+        name = routine["name"]
+        routine_id = routine["routine_id"]
+        goal = routine.get("goal", "")
+        routine_markdown = routine.get("routine_markdown", "")
+
+        print(f"[replay:routine] {name}  id={routine_id}", flush=True)
+        if goal:
+            print(f"[replay:goal] {goal}", flush=True)
+
+        # ── Validate browser UUID ─────────────────────────────────────────
+        browser_status = request_json(f"{args.url}/browsers/{args.chrome_uuid}/valid")
+        if not browser_status.get("valid", False):
+            msg = browser_status.get("message", "browser UUID is not valid")
+            print(f"Browser UUID validation failed: {msg}", file=sys.stderr)
+            return 1
+
+        # ── Create conversation in routine_replay mode ────────────────────
+        conv_result = request_json(
+            f"{args.url}/agent/conversations",
+            method="POST",
+            body={
+                "cwd": args.cwd,
+                "browser_id": args.chrome_uuid,
+                "mode": "routine_replay",
+            },
+        )
+        conversation_id = conv_result["conversation_id"]
+        print(f"[replay:conversation] {conversation_id}", flush=True)
+
+        # ── Send routine markdown as the task ────────────────────────────
+        stream_replay(
+            args.url,
+            conversation_id,
+            routine_markdown,
+            args.cwd,
+            args.chrome_uuid,
+        )
+        return 0
+
+    except URLError as exc:
+        print(f"Cannot reach OpenBrowser server: {exc}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:
+        print("Interrupted.", file=sys.stderr)
+        return 130
+    except Exception as exc:
+        print(f"Replay failed: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/skill/claude/ob-routines/scripts/start_recording.py b/skill/claude/ob-routines/scripts/start_recording.py
new file mode 100644
index 0000000..a34fad2
--- /dev/null
+++ b/skill/claude/ob-routines/scripts/start_recording.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Start a new browser recording session.
+
+The server sends a command to the Chrome extension which opens a dedicated
+recording window. After this script exits, the user performs their actions
+in that browser window. When done, they return to the terminal and run
+stop_recording.py with the printed recording_id.
+
+Example:
+  python3 start_recording.py \\
+    --chrome-uuid "$OPENBROWSER_CHROME_UUID" \\
+    --name "Gmail compose flow" \\
+    --intent "draft a new email to a contact and send it"
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from urllib.error import URLError
+from urllib.request import Request, urlopen
+
+
+def request_json(
+    url: str,
+    *,
+    method: str = "GET",
+    body: dict | None = None,
+    timeout: int = 10,
+) -> dict:
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    data = None if body is None else json.dumps(body).encode("utf-8")
+    req = Request(url, data=data, headers=headers, method=method)
+    with urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read().decode("utf-8"))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Start a new recording session in Chrome",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument(
+        "--chrome-uuid",
+        default=os.environ.get("OPENBROWSER_CHROME_UUID"),
+        help="Browser UUID capability token (or set OPENBROWSER_CHROME_UUID)",
+    )
+    parser.add_argument(
+        "--name",
+        help="Human-readable name for this recording session",
+    )
+    parser.add_argument(
+        "--intent",
+        help="Short description of what you intend to record (guides compilation later)",
+    )
+    parser.add_argument(
+        "--url",
+        default="http://127.0.0.1:8765",
+        help="OpenBrowser server URL",
+    )
+    args = parser.parse_args()
+
+    if not args.chrome_uuid:
+        print(
+            "Browser UUID is required. Set OPENBROWSER_CHROME_UUID or pass --chrome-uuid.",
+            file=sys.stderr,
+        )
+        return 2
+
+    try:
+        # Validate browser connectivity first
+        browser_status = request_json(f"{args.url}/browsers/{args.chrome_uuid}/valid")
+        if not browser_status.get("valid", False):
+            msg = browser_status.get("message", "browser UUID is not valid")
+            print(f"Browser UUID validation failed: {msg}", file=sys.stderr)
+            return 1
+
+        # Create and start recording
+        payload: dict = {"browser_id": args.chrome_uuid}
+        if args.name:
+            payload["name"] = args.name
+
+        result = request_json(f"{args.url}/recordings", method="POST", body=payload)
+        if not result.get("success"):
+            print(f"Failed to create recording: {result}", file=sys.stderr)
+            return 1
+
+        recording = result["recording"]
+        recording_id = recording["recording_id"]
+
+        # Save intent note if provided
+        if args.intent:
+            request_json(
+                f"{args.url}/recordings/{recording_id}/intent-note",
+                method="POST",
+                body={"intent_note": args.intent},
+            )
+
+    except URLError as exc:
+        print(f"Cannot reach OpenBrowser server: {exc}", file=sys.stderr)
+        return 1
+    except Exception as exc:
+        print(f"Failed to start recording: {exc}", file=sys.stderr)
+        return 1
+
+    name_display = f" ({args.name})" if args.name else ""
+    print(f"[recording:started] {recording_id}{name_display}", flush=True)
+    if args.intent:
+        print(f"[recording:intent] {args.intent}", flush=True)
+    print(
+        "\nA recording window has opened in Chrome.\n"
+        "Perform your actions in the browser, then return here and run:\n\n"
+        f"  python3 stop_recording.py {recording_id}\n",
+        flush=True,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/skill/claude/ob-routines/scripts/stop_recording.py b/skill/claude/ob-routines/scripts/stop_recording.py
new file mode 100644
index 0000000..6d91656
--- /dev/null
+++ b/skill/claude/ob-routines/scripts/stop_recording.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Stop an active recording session.
+
+Sends a stop command to the Chrome extension, which closes the recording
+window and flushes the event buffer. Prints the final event count so the
+agent knows how much was captured before kicking off compilation.
+
+Example:
+  python3 stop_recording.py abc123-recording-id
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from urllib.error import URLError
+from urllib.request import Request, urlopen
+
+
+def request_json(
+    url: str,
+    *,
+    method: str = "GET",
+    body: dict | None = None,
+    timeout: int = 15,
+) -> dict:
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    data = None if body is None else json.dumps(body).encode("utf-8")
+    req = Request(url, data=data, headers=headers, method=method)
+    with urlopen(req, timeout=timeout) as r:
+        return json.loads(r.read().decode("utf-8"))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Stop an active recording session",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument("recording_id", help="Recording ID from start_recording.py")
+    parser.add_argument(
+        "--url",
+        default="http://127.0.0.1:8765",
+        help="OpenBrowser server URL",
+    )
+    args = parser.parse_args()
+
+    try:
+        result = request_json(
+            f"{args.url}/recordings/{args.recording_id}/stop",
+            method="POST",
+            body={},
+        )
+    except URLError as exc:
+        print(f"Cannot reach OpenBrowser server: {exc}", file=sys.stderr)
+        return 1
+    except Exception as exc:
+        print(f"Failed to stop recording: {exc}", file=sys.stderr)
+        return 1
+
+    if not result.get("success"):
+        print(f"Stop failed: {result}", file=sys.stderr)
+        return 1
+
+    recording = result.get("recording") or {}
+    event_count = recording.get("event_count", "?")
+    name = recording.get("name") or ""
+    stop_reason = result.get("stop_reason", "")
+
+    display = f" ({name})" if name else ""
+    print(f"[recording:stopped] {args.recording_id}{display}", flush=True)
+    print(f"[recording:events] {event_count} events captured", flush=True)
+    if stop_reason == "browser_disconnected":
+        print(
+            "[recording:warning] Browser was disconnected — recording marked stopped "
+            "locally. Event capture may be incomplete.",
+            flush=True,
+        )
+
+    print(
+        f"\nRecording stopped. To compile this recording into a routine, run:\n\n"
+        f"  python3 compile.py {args.recording_id}\n",
+        flush=True,
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

From f4490ffcdcd618391436e1ea8d190ca968e08ae2 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Fri, 17 Apr 2026 22:26:02 +0800
Subject: [PATCH 07/12] perf(replay): auto-confirm unique clicks and trim image
 window to 1

In routine-replay mode, where the compiled SOP gives the agent precise
element keywords, the 2-phase click/select/keyboard_input confirmation
round-trip and the 3-frame screenshot history both pay for ambiguity
that does not exist.

- BrowserExecutor now tracks the most recent highlight result per
  conversation. When the agent targets the unique element that highlight
  just returned, click/select/keyboard_input skip the pending-confirmation
  round-trip and execute directly. Falls back to 2PC in any other case.
- get_context_image_window(routine_replay=True) returns 1, overriding
  the default of 3 for replay conversations only.
- ob-routines SKILL.md: tighten /ob-routines new to ask only for the
  one-line goal and defer URL/site/parameter questions to the compiler.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 server/agent/context_image_window.py   | 12 +++-
 server/agent/manager.py                |  8 ++-
 server/agent/tools/browser_executor.py | 91 ++++++++++++++++++++++++++
 skill/claude/ob-routines/SKILL.md      | 17 ++++-
 4 files changed, 124 insertions(+), 4 deletions(-)

diff --git a/server/agent/context_image_window.py b/server/agent/context_image_window.py
index c2da913..39f09cc 100644
--- a/server/agent/context_image_window.py
+++ b/server/agent/context_image_window.py
@@ -12,9 +12,16 @@
 DEFAULT_CONTEXT_IMAGE_WINDOW = 3
 
 
-def get_context_image_window() -> int | None:
+ROUTINE_REPLAY_CONTEXT_IMAGE_WINDOW = 1
+
+
+def get_context_image_window(routine_replay: bool = False) -> int | None:
     """Return the tool-image window passed to the SDK Agent.
 
+    Routine-replay conversations use a fixed window of 1: the SOP already
+    spells out each step, so a single most-recent screenshot is enough to
+    ground the next action and three-frame history would only pad context.
+
     The default is to keep only the latest screenshot-bearing tool message.
     Environment variable semantics:
     - `-1`: disable SDK filtering entirely (`None`)
@@ -22,6 +29,9 @@ def get_context_image_window() -> int | None:
     - `N >= 1`: keep the latest N screenshot-bearing tool messages
     """
 
+    if routine_replay:
+        return ROUTINE_REPLAY_CONTEXT_IMAGE_WINDOW
+
     raw_value = os.getenv(ENV_CONTEXT_IMAGE_WINDOW)
     if raw_value is None or raw_value.strip() == "":
         return DEFAULT_CONTEXT_IMAGE_WINDOW
diff --git a/server/agent/manager.py b/server/agent/manager.py
index cef99c0..7e87026 100644
--- a/server/agent/manager.py
+++ b/server/agent/manager.py
@@ -329,7 +329,9 @@ def _create_conversation_in_process(
         agent_context = self._build_agent_context()
         llm_instance = self._create_llm_from_config(model, base_url, model_alias)
         tools = self._get_tools_for_model(model, model_alias)
-        tool_image_window = get_context_image_window()
+        tool_image_window = get_context_image_window(
+            routine_replay=self._is_routine_replay_mode(mode)
+        )
         condenser_llm = llm_instance.model_copy(update={"usage_id": "condenser"})
         agent = Agent(
             llm=llm_instance,
@@ -576,7 +578,9 @@ def get_or_create_conversation(
         agent_context = self._build_agent_context()
         llm_instance = self._create_llm_from_config(model, base_url, model_alias)
         tools = self._get_tools_for_model(model, model_alias)
-        tool_image_window = get_context_image_window()
+        tool_image_window = get_context_image_window(
+            routine_replay=self._is_routine_replay_mode(mode)
+        )
         condenser_llm = llm_instance.model_copy(update={"usage_id": "condenser"})
         agent = Agent(
             llm=llm_instance,
diff --git a/server/agent/tools/browser_executor.py b/server/agent/tools/browser_executor.py
index 26b3f35..81feb97 100644
--- a/server/agent/tools/browser_executor.py
+++ b/server/agent/tools/browser_executor.py
@@ -105,6 +105,11 @@ def __init__(self):
         self.conversation_id = None
         # Pending confirmations per conversation for 2PC actions.
         self.pending_confirmations: Dict[str, Dict[str, Any]] = {}
+        # Most recent highlight result per conversation. Keyed by conversation_id,
+        # value is the list of element dicts returned by the last highlight call.
+        # Used in routine-replay mode to auto-confirm clicks/selects/keyboard_input
+        # when the target was just uniquely highlighted.
+        self.last_highlight_elements: Dict[str, List[Dict[str, Any]]] = {}
 
     def _uses_small_model(self) -> bool:
         """Whether the active conversation uses the small-model profile."""
@@ -132,6 +137,38 @@ def _uses_small_model(self) -> bool:
 
         return is_small_model(model_name)
 
+    def _is_routine_replay_mode(self) -> bool:
+        """Whether the active conversation is running in routine-replay mode."""
+        if not self.conversation_id:
+            return False
+
+        session = session_manager.get_session(str(self.conversation_id))
+        if session is None:
+            return False
+
+        return session.metadata.get("mode") == "routine_replay"
+
+    def _auto_confirm_target_id(self, requested_element_id: str) -> str | None:
+        """Return the resolved element id if auto-confirm applies, else None.
+
+        In routine-replay mode, when the most recent highlight call in this
+        conversation returned exactly one element whose id matches the one the
+        agent is now targeting, we can skip the two-phase confirmation round
+        trip: the routine SOP's precise keywords already disambiguated the
+        target, so a confirmation prompt adds latency without adding safety.
+        """
+        if not self._is_routine_replay_mode():
+            return None
+        if not self.conversation_id or not requested_element_id:
+            return None
+        recent = self.last_highlight_elements.get(self.conversation_id)
+        if not recent or len(recent) != 1:
+            return None
+        only_id = recent[0].get("id")
+        if not only_id or only_id != requested_element_id:
+            return None
+        return only_id
+
     def __call__(
         self, action: OpenBrowserAction, conversation
     ) -> OpenBrowserObservation:
@@ -333,6 +370,8 @@ def _execute_highlight_action(
         # Extract elements and pagination info
         elements = result_dict.get("data", {}).get("elements", [])
         total_elements = result_dict.get("data", {}).get("totalElements", 0)
+        if self.conversation_id:
+            self.last_highlight_elements[self.conversation_id] = list(elements)
         element_label = self._format_highlight_element_label(
             element_type=element_type, count=len(elements)
         )
@@ -366,6 +405,22 @@ def _execute_element_interaction_action(
         if action_type == "click":
             if not action.element_id:
                 raise ValueError("click requires element_id parameter")
+            auto_id = self._auto_confirm_target_id(action.element_id)
+            if auto_id:
+                command = ClickElementCommand(
+                    element_id=auto_id,
+                    conversation_id=self.conversation_id,
+                    tab_id=action.tab_id,
+                )
+                result_dict = self._execute_command_sync(command)
+                if not result_dict or not result_dict.get("success"):
+                    ext_error = self._extract_result_error(result_dict)
+                    raise RuntimeError(f"Failed to click element: {ext_error}")
+                return self._build_observation_from_result(
+                    result_dict,
+                    f"Auto-confirmed and clicked element: {auto_id}",
+                    element_id=auto_id,
+                )
             element_preview = self._get_element_full_html(action.element_id, "click")
             full_html = element_preview[0]
             screenshot = element_preview[1]
@@ -572,6 +627,23 @@ def _execute_element_interaction_action(
                 raise ValueError("keyboard_input requires element_id parameter")
             if not action.text:
                 raise ValueError("keyboard_input requires text parameter")
+            auto_id = self._auto_confirm_target_id(action.element_id)
+            if auto_id:
+                command = KeyboardInputCommand(
+                    element_id=auto_id,
+                    text=action.text,
+                    conversation_id=self.conversation_id,
+                    tab_id=action.tab_id,
+                )
+                result_dict = self._execute_command_sync(command)
+                if not result_dict or not result_dict.get("success"):
+                    ext_error = self._extract_result_error(result_dict)
+                    raise RuntimeError(f"Failed to input text: {ext_error}")
+                return self._build_observation_from_result(
+                    result_dict,
+                    f"Auto-confirmed and input text to element: {auto_id}",
+                    element_id=auto_id,
+                )
             element_preview = self._get_element_full_html(
                 action.element_id, "keyboard_input"
             )
@@ -622,6 +694,25 @@ def _execute_element_interaction_action(
                 raise ValueError("select requires element_id parameter")
             if action.value is None:
                 raise ValueError("select requires value parameter")
+            auto_id = self._auto_confirm_target_id(action.element_id)
+            if auto_id:
+                command = SelectElementCommand(
+                    element_id=auto_id,
+                    value=action.value,
+                    conversation_id=self.conversation_id,
+                    tab_id=action.tab_id,
+                )
+                result_dict = self._execute_command_sync(command)
+                if not result_dict or not result_dict.get("success"):
+                    ext_error = self._extract_result_error(result_dict)
+                    raise RuntimeError(f"Failed to select option: {ext_error}")
+                value_preview = self._format_select_value_preview(action.value)
+                return self._build_observation_from_result(
+                    result_dict,
+                    f"Auto-confirmed and selected option {value_preview} in element: "
+                    f"{auto_id}",
+                    element_id=auto_id,
+                )
             element_preview = self._get_element_full_html(action.element_id, "select")
             full_html = element_preview[0]
             screenshot = element_preview[1]
diff --git a/skill/claude/ob-routines/SKILL.md b/skill/claude/ob-routines/SKILL.md
index d9ecb7a..589bd0e 100644
--- a/skill/claude/ob-routines/SKILL.md
+++ b/skill/claude/ob-routines/SKILL.md
@@ -16,7 +16,7 @@ When invoked with arguments, act immediately — do not ask the user what they w
 |---|---|
 | `/ob-routines` | Show available routines and ask what to do |
 | `/ob-routines list [query]` | Run `list_routines.py [query]` and display results |
-| `/ob-routines new` | Ask what flow to record, then start the full record → compile pipeline |
+| `/ob-routines new` | Ask **only** for the one-line goal/intention, then start recording immediately (see "Before recording" below) |
 | `/ob-routines execute <name>` | Run `replay.py <name>` immediately |
 
 ---
@@ -116,6 +116,21 @@ python3 skill/claude/ob-routines/scripts/list_routines.py --recordings
 
 ## Record a routine
 
+### Before recording — DO NOT interrogate the user
+
+The whole point of record → compile is that the browser actions are **observed**,
+and the Compiler Agent asks clarifying questions *after* it has seen them.
+
+Ask the user **only** for a short goal/intention (one line). Do **NOT** ask:
+- which site or URL to start from
+- which tool/screener to use
+- how to define filter terms ("what's high-value?", "what's significant?")
+- which parameters should vary between runs
+
+All of that is the compiler's job during Gate 1. Pre-record interrogation
+defeats the pipeline and wastes the user's time. If the user's goal is vague
+("find good stocks"), that's fine — start recording. The compiler will ask.
+
 ### Step 1 — start recording
 ```bash
 python3 skill/claude/ob-routines/scripts/start_recording.py \

From a9c0c7c13a268f1f2d28284b024c3ce0cc9a0c81 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Sat, 18 Apr 2026 10:22:06 +0800
Subject: [PATCH 08/12] chore: apply pre-commit formatting (black) on
 ob-routines scripts

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 skill/claude/ob-routines/scripts/compile.py   | 27 ++++++++++++-------
 .../ob-routines/scripts/list_routines.py      |  8 ++----
 skill/claude/ob-routines/scripts/replay.py    | 20 +++++++-------
 3 files changed, 30 insertions(+), 25 deletions(-)

diff --git a/skill/claude/ob-routines/scripts/compile.py b/skill/claude/ob-routines/scripts/compile.py
index 5ed9a92..5f56b6e 100644
--- a/skill/claude/ob-routines/scripts/compile.py
+++ b/skill/claude/ob-routines/scripts/compile.py
@@ -29,7 +29,6 @@
 from urllib.error import HTTPError, URLError
 from urllib.request import Request, urlopen
 
-
 # ---------------------------------------------------------------------------
 # HTTP helpers
 # ---------------------------------------------------------------------------
@@ -90,10 +89,11 @@ def _format_compiler_event(event_type: str, data: dict) -> None:
             else:
                 # FileEditorTool, TraceViewerTool, SubmitWorkflowTool, etc.
                 extras = {
-                    k: v for k, v in action.items()
-                    if k != "action" and v is not None
+                    k: v for k, v in action.items() if k != "action" and v is not None
                 }
-                suffix = (" " + json.dumps(extras, ensure_ascii=False)) if extras else ""
+                suffix = (
+                    (" " + json.dumps(extras, ensure_ascii=False)) if extras else ""
+                )
                 print(f"[compiler:action] {action_name}{suffix}", flush=True)
         else:
             print(f"[compiler:action] {action}", flush=True)
@@ -173,7 +173,10 @@ def _stream_sse(url: str, body: dict) -> dict | None:
 
     except HTTPError as exc:
         body_text = exc.read().decode("utf-8", errors="replace")
-        print(f"[compiler:http_error] {exc.code} {exc.reason}: {body_text}", file=sys.stderr)
+        print(
+            f"[compiler:http_error] {exc.code} {exc.reason}: {body_text}",
+            file=sys.stderr,
+        )
         return None
 
     return complete_result
@@ -207,8 +210,10 @@ def compile_recording(base_url: str, recording_id: str, model_alias: str | None)
         if status == "asking":
             question = result.get("question", "")
             print(f"\n[compiler:question] {question}", flush=True)
-            print("[compiler:waiting_for_answer] Type your answer and press Enter:",
-                  flush=True)
+            print(
+                "[compiler:waiting_for_answer] Type your answer and press Enter:",
+                flush=True,
+            )
             try:
                 answer = input().strip()
             except (EOFError, KeyboardInterrupt):
@@ -258,7 +263,9 @@ def compile_recording(base_url: str, recording_id: str, model_alias: str | None)
             goal = result.get("goal", "")
             step_count = result.get("step_count", "?")
             routine_markdown = result.get("routine_markdown", "")
-            print(f"\n[compiler:complete] goal={goal!r}  steps={step_count}", flush=True)
+            print(
+                f"\n[compiler:complete] goal={goal!r}  steps={step_count}", flush=True
+            )
             if routine_markdown:
                 print(f"[compiler:routine_draft]\n{routine_markdown}", flush=True)
             print(
@@ -335,8 +342,7 @@ def compile_recording(base_url: str, recording_id: str, model_alias: str | None)
 
     print(f"[compiler:saved] name={name!r}  id={routine_id}  steps={steps}", flush=True)
     print(
-        f"\nRoutine saved. To replay it, run:\n\n"
-        f"  python3 replay.py {name!r}\n",
+        f"\nRoutine saved. To replay it, run:\n\n" f"  python3 replay.py {name!r}\n",
         flush=True,
     )
     return 0
@@ -345,6 +351,7 @@ def compile_recording(base_url: str, recording_id: str, model_alias: str | None)
 def _slugify(text: str) -> str:
     """Turn a goal string into a short, lowercase, hyphenated slug."""
     import re
+
     # Lowercase, keep only alnum and spaces, collapse and replace with hyphens
     slug = re.sub(r"[^\w\s]", "", text.lower())
     slug = re.sub(r"\s+", "-", slug.strip())
diff --git a/skill/claude/ob-routines/scripts/list_routines.py b/skill/claude/ob-routines/scripts/list_routines.py
index 24a1dac..a1ab1e7 100644
--- a/skill/claude/ob-routines/scripts/list_routines.py
+++ b/skill/claude/ob-routines/scripts/list_routines.py
@@ -37,8 +37,7 @@ def list_routines(base_url: str, query: str | None) -> int:
     if query:
         q = query.lower()
         items = [
-            r for r in items
-            if q in r["name"].lower() or q in r.get("goal", "").lower()
+            r for r in items if q in r["name"].lower() or q in r.get("goal", "").lower()
         ]
 
     if not items:
@@ -68,10 +67,7 @@ def list_recordings(base_url: str, query: str | None) -> int:
     items = data.get("recordings", [])
     if query:
         q = query.lower()
-        items = [
-            r for r in items
-            if q in (r.get("name") or "").lower()
-        ]
+        items = [r for r in items if q in (r.get("name") or "").lower()]
 
     if not items:
         suffix = f" matching {query!r}" if query else ""
diff --git a/skill/claude/ob-routines/scripts/replay.py b/skill/claude/ob-routines/scripts/replay.py
index 5520fb4..8b61d7b 100644
--- a/skill/claude/ob-routines/scripts/replay.py
+++ b/skill/claude/ob-routines/scripts/replay.py
@@ -20,7 +20,6 @@
 from urllib.error import URLError
 from urllib.request import Request, urlopen
 
-
 # ---------------------------------------------------------------------------
 # HTTP helpers
 # ---------------------------------------------------------------------------
@@ -77,8 +76,7 @@ def find_routine(base_url: str, query: str) -> dict | None:
 
     # 4. Substring match on name or goal
     sub = [
-        r for r in routines
-        if q in r["name"].lower() or q in r.get("goal", "").lower()
+        r for r in routines if q in r["name"].lower() or q in r.get("goal", "").lower()
     ]
     if len(sub) == 1:
         return sub[0]
@@ -191,11 +189,13 @@ def stream_replay(
 ) -> None:
     req = Request(
         f"{base_url}/agent/conversations/{conversation_id}/messages",
-        data=json.dumps({
-            "text": task,
-            "cwd": cwd,
-            "browser_id": chrome_uuid,
-        }).encode("utf-8"),
+        data=json.dumps(
+            {
+                "text": task,
+                "cwd": cwd,
+                "browser_id": chrome_uuid,
+            }
+        ).encode("utf-8"),
         headers={
             "Content-Type": "application/json",
             "Accept": "text/event-stream",
@@ -272,7 +272,9 @@ def main() -> int:
             print(f"{'NAME':<30}  {'STEPS':>5}  GOAL")
             print("-" * 72)
             for r in routines:
-                print(f"{r['name']:<30}  {r.get('step_count', '?'):>5}  {r.get('goal', '')}")
+                print(
+                    f"{r['name']:<30}  {r.get('step_count', '?'):>5}  {r.get('goal', '')}"
+                )
             return 0
 
         if not args.chrome_uuid:

From 85dd4f7007bfb8ef06c7fe555f46314326b769d8 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Sat, 18 Apr 2026 00:44:02 +0800
Subject: [PATCH 09/12] perf(highlight): cache layout reads + spatial-index
 pagination
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Tab init's two heaviest phases share the same shape: per-element loops that
re-do work the previous step already paid for. Cut both.

Scanner (highlight-detection.injected.js): wrap collectHighlightCandidates in
withScanLayoutCache, which monkey-patches Element.prototype.getBoundingClientRect,
SVGGraphicsElement.prototype.getBoundingClientRect, window.getComputedStyle, and
Document.prototype.elementsFromPoint with per-scan WeakMap/Map caches. The scan
runs in one synchronous Runtime.evaluate, so layout cannot change mid-task and
caching is safe; originals are restored in finally. Also skip inert tags
(script/style/meta/...) before the first layout read.

Pagination (collision-detection.ts): SelectedSpatialIndex (96px grid) keyed on
union(bbox, labelBBox) of placed elements. isPlacementFeasible now iterates
only nearby placed elements via nearbySelectedFor, which queries by inflate
(union(candidate.bbox, candidate.labelBBox), CLEARANCE) — covering all four
collision tests. chooseLeastBlockingPlacement also uses an "influence rect"
to skip re-evaluating spatially-far future candidates when a hypothetical
placement cannot affect them.

Measured (best run, fresh tab init):
- finviz.com (349 elements):   17.8s -> 13.7s  (-23%)
- bluebook mock (50):           6.3s -> 5.4s   (-14%)
- techforum mock (34):          4.3s -> 3.9s   (-11%)
- 16 mock sites aggregate:    -4% to -14%

Correctness:
- 181/181 extension unit tests pass.
- Strict integration check (selector + type + labelPosition + bbox + element
  ORDER) passes on all 16 deterministic mock sites.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 .../commands/highlight-detection.injected.js  | 112 ++++++++
 extension/src/utils/collision-detection.ts    | 269 ++++++++++++++++--
 2 files changed, 354 insertions(+), 27 deletions(-)

diff --git a/extension/src/commands/highlight-detection.injected.js b/extension/src/commands/highlight-detection.injected.js
index 72f3e8e..95ef232 100644
--- a/extension/src/commands/highlight-detection.injected.js
+++ b/extension/src/commands/highlight-detection.injected.js
@@ -77,6 +77,106 @@ function hasCallableMethod(value, methodNames) {
   );
 }
 
+// Layout reads (getBoundingClientRect, getComputedStyle) and elementsFromPoint
+// are the single biggest cost in collectHighlightCandidates: every visibility
+// predicate re-reads them for the same element. Within one synchronous
+// Runtime.evaluate task no page JS runs concurrently, so the values cannot
+// change mid-scan. We monkey-patch the prototypes for the duration of one
+// scan, populate a per-element WeakMap, and restore originals at the end.
+const SCAN_NON_INTERACTIVE_TAGS = new Set([
+  'script',
+  'style',
+  'link',
+  'meta',
+  'head',
+  'title',
+  'noscript',
+  'br',
+  'hr',
+  'source',
+  'track',
+  'template',
+  'param',
+  'col',
+  'colgroup',
+]);
+
+function isScanSkippableTag(el) {
+  if (!el || !el.tagName) return false;
+  return SCAN_NON_INTERACTIVE_TAGS.has(el.tagName.toLowerCase());
+}
+
+function withScanLayoutCache(fn) {
+  const rectCache = new WeakMap();
+  const styleCache = new WeakMap();
+  // elementsFromPoint dedup keyed by rounded "x:y"
+  const efpCache = new Map();
+
+  const origElementRect = Element.prototype.getBoundingClientRect;
+  const SVGGraphicsProto =
+    typeof SVGGraphicsElement !== 'undefined'
+      ? SVGGraphicsElement.prototype
+      : null;
+  const origSVGRect =
+    SVGGraphicsProto && SVGGraphicsProto.getBoundingClientRect;
+  const origGetComputedStyle = window.getComputedStyle;
+  // Patch Document.prototype rather than the document instance so we don't
+  // leave an own-property shadowing the prototype after the scan finishes.
+  const DocumentProto =
+    typeof Document !== 'undefined' ? Document.prototype : null;
+  const origElementsFromPoint =
+    DocumentProto && DocumentProto.elementsFromPoint;
+
+  function patchedRect() {
+    let r = rectCache.get(this);
+    if (r === undefined) {
+      r = origElementRect.call(this);
+      rectCache.set(this, r);
+    }
+    return r;
+  }
+
+  Element.prototype.getBoundingClientRect = patchedRect;
+  if (SVGGraphicsProto && origSVGRect) {
+    SVGGraphicsProto.getBoundingClientRect = patchedRect;
+  }
+
+  window.getComputedStyle = function (el, pseudo) {
+    if (pseudo) return origGetComputedStyle.call(window, el, pseudo);
+    let s = styleCache.get(el);
+    if (s === undefined) {
+      s = origGetComputedStyle.call(window, el);
+      styleCache.set(el, s);
+    }
+    return s;
+  };
+
+  if (DocumentProto && origElementsFromPoint) {
+    DocumentProto.elementsFromPoint = function (x, y) {
+      const key = Math.round(x) + ':' + Math.round(y);
+      let stack = efpCache.get(key);
+      if (stack === undefined) {
+        stack = origElementsFromPoint.call(this, x, y);
+        efpCache.set(key, stack);
+      }
+      return stack;
+    };
+  }
+
+  try {
+    return fn();
+  } finally {
+    Element.prototype.getBoundingClientRect = origElementRect;
+    if (SVGGraphicsProto && origSVGRect) {
+      SVGGraphicsProto.getBoundingClientRect = origSVGRect;
+    }
+    window.getComputedStyle = origGetComputedStyle;
+    if (DocumentProto && origElementsFromPoint) {
+      DocumentProto.elementsFromPoint = origElementsFromPoint;
+    }
+  }
+}
+
 function createHighlightTrace() {
   const traceStart = performance.now();
 
@@ -2473,6 +2573,12 @@ function collectUploadableCandidates(trace) {
 }
 
 function collectHighlightCandidates(config, trace, layoutStability) {
+  return withScanLayoutCache(() =>
+    collectHighlightCandidatesImpl(config, trace, layoutStability),
+  );
+}
+
+function collectHighlightCandidatesImpl(config, trace, layoutStability) {
   const activeTopLayerRoot = getActiveTopLayerRoot();
   const registry = new Map();
 
@@ -2529,6 +2635,12 @@ function collectHighlightCandidates(config, trace, layoutStability) {
       );
     }
 
+    // Tag-only fast reject before any layout read. Saves rect/style work on
+    // the long tail of inert markup (script/style/meta/...).
+    if (isScanSkippableTag(element)) {
+      continue;
+    }
+
     if (!isElementInViewportForDetection(element)) {
       continue;
     }
diff --git a/extension/src/utils/collision-detection.ts b/extension/src/utils/collision-detection.ts
index 054abc2..da4911a 100644
--- a/extension/src/utils/collision-detection.ts
+++ b/extension/src/utils/collision-detection.ts
@@ -36,6 +36,108 @@ interface RemainingCandidate {
   element: InteractiveElement;
 }
 
+// Coarse spatial grid used to skip O(N) scans of `selected` and `remaining`
+// when checking collisions. Cell size is a heuristic — large enough that most
+// label rects touch only a couple of cells, small enough that a typical
+// query returns far fewer than the full set.
+const SPATIAL_INDEX_CELL_PX = 96;
+
+class SelectedSpatialIndex {
+  private cells = new Map<number, InteractiveElement[]>();
+
+  add(element: InteractiveElement): void {
+    const labelBBox = getLabelBBox(
+      element.bbox,
+      element.labelPosition ?? 'above',
+      element.id,
+    );
+    const union = unionBBox(element.bbox, labelBBox);
+    this.forEachCell(union, (key) => {
+      let bucket = this.cells.get(key);
+      if (!bucket) {
+        bucket = [];
+        this.cells.set(key, bucket);
+      }
+      // Avoid duplicate registration when a single element straddles cells we
+      // visit out of order — the per-call dedup Set in queryNear handles dup
+      // results across cells.
+      if (bucket[bucket.length - 1] !== element) {
+        bucket.push(element);
+      }
+    });
+  }
+
+  // Returns elements whose registered union-rect lies in any cell touched by
+  // the query rect (inflated by clearance on each side). Includes elements
+  // whose registration cells are *adjacent* to the query rect — see
+  // `queryNear` callers, which already inflate the query rect with clearance.
+  queryNear(query: BBox): InteractiveElement[] {
+    const seen = new Set<InteractiveElement>();
+    const out: InteractiveElement[] = [];
+    this.forEachCell(query, (key) => {
+      const bucket = this.cells.get(key);
+      if (!bucket) return;
+      for (const el of bucket) {
+        if (!seen.has(el)) {
+          seen.add(el);
+          out.push(el);
+        }
+      }
+    });
+    return out;
+  }
+
+  private forEachCell(rect: BBox, fn: (key: number) => void): void {
+    // Real bboxes from getBoundingClientRect are always finite, but synthetic
+    // test inputs or future callers might pass NaN/Infinity. Without this
+    // guard Math.floor would yield NaN, the loop would skip, and we'd
+    // silently drop a registration — masking real collisions.
+    if (
+      !Number.isFinite(rect.x) ||
+      !Number.isFinite(rect.y) ||
+      !Number.isFinite(rect.width) ||
+      !Number.isFinite(rect.height)
+    ) {
+      // Single sentinel cell so the registration is still discoverable.
+      fn(Number.MIN_SAFE_INTEGER);
+      return;
+    }
+    const minCx = Math.floor(rect.x / SPATIAL_INDEX_CELL_PX);
+    const maxCx = Math.floor(
+      (rect.x + Math.max(0, rect.width)) / SPATIAL_INDEX_CELL_PX,
+    );
+    const minCy = Math.floor(rect.y / SPATIAL_INDEX_CELL_PX);
+    const maxCy = Math.floor(
+      (rect.y + Math.max(0, rect.height)) / SPATIAL_INDEX_CELL_PX,
+    );
+    for (let cy = minCy; cy <= maxCy; cy++) {
+      for (let cx = minCx; cx <= maxCx; cx++) {
+        // Cantor-pair-ish key: cy gets the high bits, cx the low bits.
+        // Negative coords are uncommon for label rects but still encode safely
+        // because Math.floor preserves order under shift.
+        fn(cy * 100000 + cx);
+      }
+    }
+  }
+}
+
+function unionBBox(a: BBox, b: BBox): BBox {
+  const x = Math.min(a.x, b.x);
+  const y = Math.min(a.y, b.y);
+  const xMax = Math.max(a.x + a.width, b.x + b.width);
+  const yMax = Math.max(a.y + a.height, b.y + b.height);
+  return { x, y, width: xMax - x, height: yMax - y };
+}
+
+function inflateBBox(rect: BBox, padding: number): BBox {
+  return {
+    x: rect.x - padding,
+    y: rect.y - padding,
+    width: rect.width + 2 * padding,
+    height: rect.height + 2 * padding,
+  };
+}
+
 interface PlacementEvaluation {
   position: LabelPosition;
   blockedCandidateCount: number;
@@ -302,12 +404,14 @@ function buildCollisionFreePages(
 
   while (remaining.length > 0) {
     const selected: InteractiveElement[] = [];
+    const selectedIndex = new SelectedSpatialIndex();
     let pageRemaining = remaining;
 
     while (pageRemaining.length > 0) {
       const nextSelection = chooseNextCandidate(
         pageRemaining,
         selected,
+        selectedIndex,
         viewportWidth,
         viewportHeight,
       );
@@ -316,10 +420,12 @@ function buildCollisionFreePages(
         break;
       }
 
-      selected.push({
+      const placed: InteractiveElement = {
         ...nextSelection.candidate.element,
         labelPosition: nextSelection.position,
-      });
+      };
+      selected.push(placed);
+      selectedIndex.add(placed);
       pageRemaining = pageRemaining.filter(
         (candidate) =>
           candidate.sourceIndex !== nextSelection.candidate.sourceIndex,
@@ -347,14 +453,16 @@ function tryBuildUniformPositionPage(
   viewportHeight?: number,
 ): InteractiveElement[] | null {
   const selected: InteractiveElement[] = [];
+  const index = new SelectedSpatialIndex();
 
   for (const element of elements) {
+    const nearby = nearbySelectedFor(element, position, element.id, index);
     if (
       !isPlacementFeasible(
         element,
         element.id,
         position,
-        selected,
+        nearby,
         viewportWidth,
         viewportHeight,
       )
@@ -362,10 +470,12 @@ function tryBuildUniformPositionPage(
       return null;
     }
 
-    selected.push({
+    const placed: InteractiveElement = {
       ...element,
       labelPosition: position,
-    });
+    };
+    selected.push(placed);
+    index.add(placed);
   }
 
   return selected;
@@ -374,6 +484,7 @@ function tryBuildUniformPositionPage(
 function chooseNextCandidate(
   remaining: RemainingCandidate[],
   selected: InteractiveElement[],
+  selectedIndex: SelectedSpatialIndex,
   viewportWidth?: number,
   viewportHeight?: number,
 ): (PlacementEvaluation & { candidate: RemainingCandidate }) | null {
@@ -388,6 +499,7 @@ function chooseNextCandidate(
       candidate.element,
       candidate.element.id,
       selected,
+      selectedIndex,
       viewportWidth,
       viewportHeight,
     );
@@ -415,6 +527,7 @@ function chooseNextCandidate(
       constrainedCandidate.feasiblePositions,
       remaining,
       selected,
+      selectedIndex,
       viewportWidth,
       viewportHeight,
     ),
@@ -426,6 +539,7 @@ function chooseLeastBlockingPlacement(
   feasiblePositions: LabelPosition[],
   remaining: RemainingCandidate[],
   selected: InteractiveElement[],
+  selectedIndex: SelectedSpatialIndex,
   viewportWidth?: number,
   viewportHeight?: number,
 ): PlacementEvaluation {
@@ -435,31 +549,106 @@ function chooseLeastBlockingPlacement(
   );
   let bestPlacement: PlacementEvaluation | null = null;
 
+  // Pre-compute each future candidate's baseline feasible positions against
+  // the current `selected` set. When we test a hypothetical placement of
+  // `candidate@position`, only future candidates whose bbox/label is
+  // geometrically near that placement can have their feasibility change. The
+  // rest keep their baseline feasibility — saving the O(|future|×4×|selected|)
+  // recomputation per position.
+  interface FutureBaseline {
+    candidate: RemainingCandidate;
+    elementUnion: BBox; // bbox ∪ all four label rects
+    feasibleCount: number;
+    totalLength: number;
+  }
+  const futureBaselines: FutureBaseline[] = futureCandidates.map((fc) => {
+    const baseline = getFeasiblePositions(
+      fc.element,
+      fc.element.id,
+      selected,
+      selectedIndex,
+      viewportWidth,
+      viewportHeight,
+    );
+    let union = fc.element.bbox;
+    for (const pos of POSITION_PRIORITY) {
+      union = unionBBox(union, getLabelBBox(fc.element.bbox, pos, fc.element.id));
+    }
+    return {
+      candidate: fc,
+      elementUnion: union,
+      feasibleCount: baseline.length,
+      totalLength: baseline.length,
+    };
+  });
+
+  const baselineBlockedCount = futureBaselines.reduce(
+    (acc, fb) => (fb.feasibleCount === 0 ? acc + 1 : acc),
+    0,
+  );
+  const baselineTotalOptions = futureBaselines.reduce(
+    (acc, fb) => acc + fb.totalLength,
+    0,
+  );
+
   for (const position of feasiblePositions) {
-    const hypotheticalSelected = [
-      ...selected,
-      {
-        ...candidate.element,
-        labelPosition: position,
-      },
-    ];
-    let blockedCandidateCount = 0;
-    let totalFutureOptions = 0;
-
-    futureCandidates.forEach((candidate) => {
-      const futureOptions = getFeasiblePositions(
-        candidate.element,
-        candidate.element.id,
-        hypotheticalSelected,
-        viewportWidth,
-        viewportHeight,
-      );
+    const hypotheticalElement: InteractiveElement = {
+      ...candidate.element,
+      labelPosition: position,
+    };
+    const hypotheticalLabelBBox = getLabelBBox(
+      candidate.element.bbox,
+      position,
+      candidate.element.id,
+    );
+    // Influence rect: anything whose elementUnion does NOT intersect this
+    // (inflated by clearance) cannot be affected by adding the hypothetical
+    // candidate. We only need to recompute for future candidates inside it.
+    const influenceRect = inflateBBox(
+      unionBBox(candidate.element.bbox, hypotheticalLabelBBox),
+      VISUAL_LABEL_CLEARANCE_PX,
+    );
 
-      if (futureOptions.length === 0) {
+    let blockedCandidateCount = baselineBlockedCount;
+    let totalFutureOptions = baselineTotalOptions;
+
+    for (const fb of futureBaselines) {
+      if (!bboxesIntersect(fb.elementUnion, influenceRect)) {
+        continue;
+      }
+      // Feasibility can change for this future candidate. Re-test against
+      // the spatially-near selected set plus the hypothetical candidate.
+      let updatedFeasibleLen = 0;
+      for (const pos of POSITION_PRIORITY) {
+        const nearby = nearbySelectedFor(
+          fb.candidate.element,
+          pos,
+          fb.candidate.element.id,
+          selectedIndex,
+          [hypotheticalElement],
+        );
+        if (
+          isPlacementFeasible(
+            fb.candidate.element,
+            fb.candidate.element.id,
+            pos,
+            nearby,
+            viewportWidth,
+            viewportHeight,
+          )
+        ) {
+          updatedFeasibleLen++;
+        }
+      }
+
+      // Adjust baseline aggregates for the delta on this single future.
+      if (fb.feasibleCount === 0 && updatedFeasibleLen > 0) {
+        blockedCandidateCount--;
+      } else if (fb.feasibleCount > 0 && updatedFeasibleLen === 0) {
         blockedCandidateCount++;
       }
-      totalFutureOptions += futureOptions.length;
-    });
+      totalFutureOptions += updatedFeasibleLen - fb.totalLength;
+    }
 
     if (
       !bestPlacement ||
@@ -492,18 +681,22 @@ function getFeasiblePositions(
   element: InteractiveElement,
   labelText: string,
   selected: InteractiveElement[],
+  selectedIndex: SelectedSpatialIndex | null,
   viewportWidth?: number,
   viewportHeight?: number,
 ): LabelPosition[] {
   const feasiblePositions: LabelPosition[] = [];
 
   for (const position of POSITION_PRIORITY) {
+    const nearby = selectedIndex
+      ? nearbySelectedFor(element, position, labelText, selectedIndex)
+      : selected;
     if (
       isPlacementFeasible(
         element,
         labelText,
         position,
-        selected,
+        nearby,
         viewportWidth,
         viewportHeight,
       )
@@ -515,6 +708,28 @@ function getFeasiblePositions(
   return feasiblePositions;
 }
 
+// Returns the subset of `selected` that could plausibly collide with the
+// candidate placement. The query rect is the union of the candidate's bbox
+// and its label rect for the requested position, inflated by the visible
+// clearance threshold. Optional `extras` are appended (e.g. a hypothetical
+// candidate not yet inserted into the index).
+function nearbySelectedFor(
+  element: InteractiveElement,
+  position: LabelPosition,
+  labelText: string,
+  index: SelectedSpatialIndex,
+  extras: InteractiveElement[] = [],
+): InteractiveElement[] {
+  const labelBBox = getLabelBBox(element.bbox, position, labelText);
+  const query = inflateBBox(
+    unionBBox(element.bbox, labelBBox),
+    VISUAL_LABEL_CLEARANCE_PX,
+  );
+  const near = index.queryNear(query);
+  if (extras.length === 0) return near;
+  return near.concat(extras);
+}
+
 function isPlacementFeasible(
   element: InteractiveElement,
   labelText: string,

From 9e9cc021fab4e155f67376f41b8ff18a7bbc7ea7 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Sat, 18 Apr 2026 01:18:07 +0800
Subject: [PATCH 10/12] perf(highlight): cache resolve-phase classifiers + emit
 perf breakdown
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The pagination win revealed that scan-phase resolve was the new bottleneck:
on finviz the resolve phase alone was 6.1s of a 6.5s scan. Per candidate,
resolveClickableCandidate walks up to 5 ancestors, each calling
isClickableCandidate, which calls hasExplicitClickableAncestor that walks
ALL ancestors back to body, calling getSemanticClickableSignal at each.
For deep DOM (finviz tables) the same elements were classified dozens of
times per scan.

Add per-scan WeakMap memoization (cleared by withScanLayoutCache) for the
classifiers that are pure functions of element + DOM state:
- getSemanticClickableSignal
- isClickableCandidate
- getBaseClickableSignal
- hasExplicitClickableAncestor
- getElementTextForDetection (textContent walk)
- getElementSearchText

Also add scan_stats / scan_times to the response payload so harness/tooling
can attribute time per phase without parsing console output.

Measured (best run, finviz.com/screener.ashx, ~349 candidates):
- in-page scan:    6537ms -> 585ms     (-91%, ~11x)
- pagination:       397ms -> 300ms     (already optimized in prior commit)
- end-to-end:    17787ms -> 4975ms     (-72%, ~3.6x)

Resolve-phase breakdown after caching: 6121ms -> 51ms.

Correctness: 181/181 unit tests pass. Strict integration check (selector,
type, labelPosition, bbox, element ORDER) passes on all 16 deterministic
mock sites — same elements, same labels, same order. finviz_real returns
identical 336/6/138 element/page/page1 counts.

Caching is safe because the scan runs in one synchronous Runtime.evaluate
call and these classifiers depend only on DOM state that cannot mutate
during the scan.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 extension/src/background/index.ts             |  21 ++-
 .../commands/highlight-detection.injected.js  | 170 ++++++++++++++++--
 2 files changed, 175 insertions(+), 16 deletions(-)

diff --git a/extension/src/background/index.ts b/extension/src/background/index.ts
index b35132c..5228fd9 100644
--- a/extension/src/background/index.ts
+++ b/extension/src/background/index.ts
@@ -583,7 +583,11 @@ async function captureHighlightedPageState(
         : '';
     const detectedViewport = detectionResult.result.value.viewport || {};
     const layoutStability = detectionResult.result.value.layoutStability;
+    const inPagePerf = detectionResult.result.value._perf || {};
     const highlightTraceStart = Date.now();
+    let paginationMs = 0;
+    let screenshotMs = 0;
+    let consistencyMs = 0;
     const detectedViewportWidth =
       typeof detectedViewport.width === 'number' ? detectedViewport.width : 0;
     const detectedViewportHeight =
@@ -656,8 +660,9 @@ async function captureHighlightedPageState(
       console.log(
         `📄 [${logLabel}] Page ${page}/${totalPages}, showing ${paginatedElements.length} of ${filteredElements.length} elements`,
       );
+      paginationMs = Date.now() - paginationBuildStart;
       console.log(
-        `⏱️ [HighlightTrace] background pagination build-pages=${Date.now() - paginationBuildStart}ms (page=${page}, viewport=${detectedViewportWidth}x${detectedViewportHeight})`,
+        `⏱️ [HighlightTrace] background pagination build-pages=${paginationMs}ms (page=${page}, viewport=${detectedViewportWidth}x${detectedViewportHeight})`,
       );
     }
 
@@ -702,8 +707,9 @@ async function captureHighlightedPageState(
     console.log(
       `📸 [${logLabel}] Screenshot captured (with in-page highlights), size: ${screenshotResult.imageData.length} bytes`,
     );
+    screenshotMs = Date.now() - screenshotStart;
     console.log(
-      `⏱️ [HighlightTrace] background screenshot ${Date.now() - screenshotStart}ms`,
+      `⏱️ [HighlightTrace] background screenshot ${screenshotMs}ms`,
     );
 
     // Apply bboxes returned from the highlight injection script
@@ -766,8 +772,9 @@ async function captureHighlightedPageState(
         })),
       currentConsistencySamples,
     );
+    consistencyMs = Date.now() - consistencyCheckStart;
     console.log(
-      `⏱️ [HighlightTrace] background consistency-check ${Date.now() - consistencyCheckStart}ms (checked=${highlightConsistency.checkedCount}, matched=${highlightConsistency.matchedCount}, missing=${highlightConsistency.missingCount}, shifted=${highlightConsistency.shiftedCount}, maxCenterShift=${highlightConsistency.maxCenterShift}, maxSizeDelta=${highlightConsistency.maxSizeDelta}, retry=${highlightConsistency.shouldRetry})`,
+      `⏱️ [HighlightTrace] background consistency-check ${consistencyMs}ms (checked=${highlightConsistency.checkedCount}, matched=${highlightConsistency.matchedCount}, missing=${highlightConsistency.missingCount}, shifted=${highlightConsistency.shiftedCount}, maxCenterShift=${highlightConsistency.maxCenterShift}, maxSizeDelta=${highlightConsistency.maxSizeDelta}, retry=${highlightConsistency.shouldRetry})`,
     );
     const repeatedDrift = isRepeatedHighlightDrift(
       highlightConsistency,
@@ -841,6 +848,14 @@ async function captureHighlightedPageState(
       page: currentPage,
       pageState,
       readinessReasons,
+      _perf: {
+        scan_ms: typeof inPagePerf.scan_ms === 'number' ? inPagePerf.scan_ms : 0,
+        scan_stats: inPagePerf.scan_stats || {},
+        scan_times: inPagePerf.scan_times || {},
+        pagination_ms: paginationMs,
+        screenshot_ms: screenshotMs,
+        consistency_ms: consistencyMs,
+      },
       ...buildScreenshotPayload(compressedScreenshotResult),
     };
   }
diff --git a/extension/src/commands/highlight-detection.injected.js b/extension/src/commands/highlight-detection.injected.js
index 95ef232..9581d94 100644
--- a/extension/src/commands/highlight-detection.injected.js
+++ b/extension/src/commands/highlight-detection.injected.js
@@ -106,11 +106,29 @@ function isScanSkippableTag(el) {
   return SCAN_NON_INTERACTIVE_TAGS.has(el.tagName.toLowerCase());
 }
 
+// Per-scan memoization caches for pure-function classifiers that get hit many
+// times for the same element during the resolve phase (each candidate walks
+// up to 5 ancestors, each ancestor calls hasExplicitClickableAncestor which
+// walks ALL ancestors, etc.). Reset at the start of each scan, leak nothing
+// outside it. WeakMap so any GC'd nodes drop out automatically.
+let _scanSemanticSignalCache = null;
+let _scanClickableCandidateCache = null;
+let _scanBaseClickableSignalCache = null;
+let _scanTextContentCache = null;
+let _scanSearchTextCache = null;
+let _scanExplicitAncestorCache = null;
+
 function withScanLayoutCache(fn) {
   const rectCache = new WeakMap();
   const styleCache = new WeakMap();
   // elementsFromPoint dedup keyed by rounded "x:y"
   const efpCache = new Map();
+  _scanSemanticSignalCache = new WeakMap();
+  _scanClickableCandidateCache = new WeakMap();
+  _scanBaseClickableSignalCache = new WeakMap();
+  _scanTextContentCache = new WeakMap();
+  _scanSearchTextCache = new WeakMap();
+  _scanExplicitAncestorCache = new WeakMap();
 
   const origElementRect = Element.prototype.getBoundingClientRect;
   const SVGGraphicsProto =
@@ -174,6 +192,12 @@ function withScanLayoutCache(fn) {
     if (DocumentProto && origElementsFromPoint) {
       DocumentProto.elementsFromPoint = origElementsFromPoint;
     }
+    _scanSemanticSignalCache = null;
+    _scanClickableCandidateCache = null;
+    _scanBaseClickableSignalCache = null;
+    _scanTextContentCache = null;
+    _scanSearchTextCache = null;
+    _scanExplicitAncestorCache = null;
   }
 }
 
@@ -405,6 +429,15 @@ function getSwipeMarkerText(el) {
 }
 
 function getElementTextForDetection(el) {
+  if (_scanTextContentCache && _scanTextContentCache.has(el)) {
+    return _scanTextContentCache.get(el);
+  }
+  const r = getElementTextForDetectionImpl(el);
+  if (_scanTextContentCache) _scanTextContentCache.set(el, r);
+  return r;
+}
+
+function getElementTextForDetectionImpl(el) {
   if (el instanceof HTMLInputElement) {
     const inputType = (el.type || '').toLowerCase();
     if (
@@ -416,10 +449,22 @@ function getElementTextForDetection(el) {
     }
   }
 
+  // textContent on a deep node walks the entire subtree of text nodes — for
+  // a table row with hundreds of descendants this is expensive enough to
+  // dominate the resolve phase. Cache so each candidate pays at most once.
   return normalizeWhitespace(el.textContent || '', 240);
 }
 
 function getElementSearchText(el) {
+  if (_scanSearchTextCache && _scanSearchTextCache.has(el)) {
+    return _scanSearchTextCache.get(el);
+  }
+  const r = getElementSearchTextImpl(el);
+  if (_scanSearchTextCache) _scanSearchTextCache.set(el, r);
+  return r;
+}
+
+function getElementSearchTextImpl(el) {
   const tokens = [
     el.tagName.toLowerCase(),
     ...getAttributeTextTokens(el, [
@@ -580,6 +625,15 @@ function hasPointerCursor(el) {
 }
 
 function getBaseClickableSignal(el) {
+  if (_scanBaseClickableSignalCache && _scanBaseClickableSignalCache.has(el)) {
+    return _scanBaseClickableSignalCache.get(el);
+  }
+  const r = getBaseClickableSignalImpl(el);
+  if (_scanBaseClickableSignalCache) _scanBaseClickableSignalCache.set(el, r);
+  return r;
+}
+
+function getBaseClickableSignalImpl(el) {
   const semanticSignal = getSemanticClickableSignal(el);
   if (semanticSignal) {
     return semanticSignal;
@@ -673,6 +727,15 @@ function getControlAffinityScore(el) {
 }
 
 function getSemanticClickableSignal(el) {
+  if (_scanSemanticSignalCache && _scanSemanticSignalCache.has(el)) {
+    return _scanSemanticSignalCache.get(el);
+  }
+  const r = getSemanticClickableSignalImpl(el);
+  if (_scanSemanticSignalCache) _scanSemanticSignalCache.set(el, r);
+  return r;
+}
+
+function getSemanticClickableSignalImpl(el) {
   const tag = el.tagName.toLowerCase();
   const role = (el.getAttribute('role') || '').toLowerCase();
 
@@ -869,18 +932,30 @@ function countDirectClickableChildren(el) {
 }
 
 function hasExplicitClickableAncestor(el) {
+  if (_scanExplicitAncestorCache && _scanExplicitAncestorCache.has(el)) {
+    return _scanExplicitAncestorCache.get(el);
+  }
+  // Per-call top-level memoization only. A previous version tried to
+  // walk-and-memoize each visited ancestor too, but that's incorrect —
+  // a node's own `hasExplicitClickableAncestor` is about ITS ancestors,
+  // not about its own signal, and it's also influenced by its own signal
+  // when answering the same question for *its* descendants. Doing the full
+  // walk per unique element (with getSemanticClickableSignal cached) is
+  // already cheap enough thanks to the upstream caches.
   let current = el.parentElement;
-
+  let answer = false;
   while (current && current !== document.body) {
     const signal = getSemanticClickableSignal(current);
     if (signal === 'semantic' || signal === 'attribute') {
-      return true;
+      answer = true;
+      break;
     }
-
     current = current.parentElement;
   }
-
-  return false;
+  if (_scanExplicitAncestorCache) {
+    _scanExplicitAncestorCache.set(el, answer);
+  }
+  return answer;
 }
 
 function isInputableCandidate(el) {
@@ -1011,6 +1086,15 @@ function hasStructuredInteractiveDescendant(el) {
 }
 
 function isClickableCandidate(el) {
+  if (_scanClickableCandidateCache && _scanClickableCandidateCache.has(el)) {
+    return _scanClickableCandidateCache.get(el);
+  }
+  const r = isClickableCandidateImpl(el);
+  if (_scanClickableCandidateCache) _scanClickableCandidateCache.set(el, r);
+  return r;
+}
+
+function isClickableCandidateImpl(el) {
   if (isDisabledForDetection(el)) {
     return null;
   }
@@ -2625,6 +2709,27 @@ function collectHighlightCandidatesImpl(config, trace, layoutStability) {
   );
 
   let scannedCount = 0;
+  // Per-phase reject counters and timings — gated behind the trace, helps
+  // identify where the scan budget is spent without per-element console spam.
+  const phaseStats = {
+    tagSkip: 0,
+    notInViewport: 0,
+    notVisible: 0,
+    scrollParentClipped: 0,
+    notInActiveTopLayer: 0,
+    hitTestOccluded: 0,
+    notResolvable: 0,
+    matched: 0,
+  };
+  const phaseTimes = {
+    tag: 0,
+    viewport: 0,
+    visible: 0,
+    scrollParent: 0,
+    topLayer: 0,
+    hitTest: 0,
+    resolve: 0,
+  };
   for (const element of allElements) {
     scannedCount += 1;
 
@@ -2635,40 +2740,65 @@ function collectHighlightCandidatesImpl(config, trace, layoutStability) {
       );
     }
 
-    // Tag-only fast reject before any layout read. Saves rect/style work on
-    // the long tail of inert markup (script/style/meta/...).
+    let t = performance.now();
     if (isScanSkippableTag(element)) {
+      phaseStats.tagSkip += 1;
+      phaseTimes.tag += performance.now() - t;
       continue;
     }
+    phaseTimes.tag += performance.now() - t;
 
-    if (!isElementInViewportForDetection(element)) {
+    t = performance.now();
+    const inViewport = isElementInViewportForDetection(element);
+    phaseTimes.viewport += performance.now() - t;
+    if (!inViewport) {
+      phaseStats.notInViewport += 1;
       continue;
     }
 
-    if (!isElementVisibleForDetection(element)) {
+    t = performance.now();
+    const visible = isElementVisibleForDetection(element);
+    phaseTimes.visible += performance.now() - t;
+    if (!visible) {
+      phaseStats.notVisible += 1;
       continue;
     }
 
-    if (!isElementVisibleInScrollParent(element)) {
+    t = performance.now();
+    const scrollOk = isElementVisibleInScrollParent(element);
+    phaseTimes.scrollParent += performance.now() - t;
+    if (!scrollOk) {
+      phaseStats.scrollParentClipped += 1;
       continue;
     }
 
-    if (!isElementInActiveTopLayer(element, activeTopLayerRoot)) {
+    t = performance.now();
+    const topLayerOk = isElementInActiveTopLayer(element, activeTopLayerRoot);
+    phaseTimes.topLayer += performance.now() - t;
+    if (!topLayerOk) {
+      phaseStats.notInActiveTopLayer += 1;
       continue;
     }
 
+    t = performance.now();
     const hitTestVisibility = getElementHitTestVisibility(element);
+    phaseTimes.hitTest += performance.now() - t;
     if (!hitTestVisibility.visible) {
+      phaseStats.hitTestOccluded += 1;
       continue;
     }
 
+    t = performance.now();
     const resolvedCandidate = resolveElementCandidate(
       element,
       config.elementType,
     );
+    phaseTimes.resolve += performance.now() - t;
     if (!resolvedCandidate) {
+      phaseStats.notResolvable += 1;
       continue;
     }
+    phaseStats.matched += 1;
 
     const candidate = {
       element: resolvedCandidate.element,
@@ -2717,14 +2847,20 @@ function collectHighlightCandidatesImpl(config, trace, layoutStability) {
     return element;
   });
 
+  const roundedTimes = {};
+  for (const k of Object.keys(phaseTimes)) {
+    roundedTimes[k] = Math.round(phaseTimes[k]);
+  }
   trace(
     'scan:done',
-    `processed=${scannedCount} matched=${elements.length} counts=${JSON.stringify(counts)}`,
+    `processed=${scannedCount} matched=${elements.length} counts=${JSON.stringify(counts)} reject=${JSON.stringify(phaseStats)} ms=${JSON.stringify(roundedTimes)}`,
   );
 
   return {
     elements,
     counts,
+    _scan_stats: phaseStats,
+    _scan_times: roundedTimes,
   };
 }
 
@@ -2737,11 +2873,14 @@ async function runOpenBrowserHighlightDetection(config) {
 
   const layoutStability = evaluateReadinessSnapshot(trace);
 
-  const { elements, counts } = collectHighlightCandidates(
+  const scanStart = performance.now();
+  const scanResult = collectHighlightCandidates(
     config,
     trace,
     layoutStability,
   );
+  const { elements, counts } = scanResult;
+  const scanMs = Math.round(performance.now() - scanStart);
 
   trace('return', `elements=${elements.length}`);
   return {
@@ -2753,5 +2892,10 @@ async function runOpenBrowserHighlightDetection(config) {
       width: window.innerWidth,
       height: window.innerHeight,
     },
+    _perf: {
+      scan_ms: scanMs,
+      scan_stats: scanResult._scan_stats || {},
+      scan_times: scanResult._scan_times || {},
+    },
   };
 }

From e8268a86605b8d34428bec018cdcee774bb15fbb Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Sat, 18 Apr 2026 10:06:28 +0800
Subject: [PATCH 11/12] set extension auto reload timeout to 40s

---
 extension/vite.config.ts | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/extension/vite.config.ts b/extension/vite.config.ts
index bf660fc..3ebf0bd 100644
--- a/extension/vite.config.ts
+++ b/extension/vite.config.ts
@@ -122,16 +122,18 @@ const devReloadPlugin = () => {
         return;
       }
 
-      // Otherwise wait for the extension to connect (up to 10s)
+      // Otherwise wait for the extension to connect (up to 40s — covers a
+      // full chrome.alarms keepalive cycle when the MV3 service worker has
+      // been terminated by Chrome).
       console.log(
         '🔄 [DevReload] Build complete — waiting for extension to connect...',
       );
       const timeout = setTimeout(() => {
         console.warn(
-          '🔄 [DevReload] No extension connected within 10s. Reload the extension manually once, then future `npm run dev` runs will auto-reload.',
+          '🔄 [DevReload] No extension connected within 40s. Reload the extension manually once, then future `npm run dev` runs will auto-reload.',
         );
         process.exit(0);
-      }, 10_000);
+      }, 40_000);
 
       // Check periodically if a client has connected
       const poll = setInterval(() => {

From 6aac696f972b606665e28450904a0d4193ab8547 Mon Sep 17 00:00:00 2001
From: Xiao Yang <yangxiao1098@gmail.com>
Date: Sat, 18 Apr 2026 10:22:17 +0800
Subject: [PATCH 12/12] chore: apply pre-commit formatting (prettier) on
 highlight perf changes

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 extension/src/background/index.ts                      | 7 +++----
 extension/src/commands/highlight-detection.injected.js | 6 +-----
 extension/src/utils/collision-detection.ts             | 5 ++++-
 3 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/extension/src/background/index.ts b/extension/src/background/index.ts
index 5228fd9..3b0f24b 100644
--- a/extension/src/background/index.ts
+++ b/extension/src/background/index.ts
@@ -708,9 +708,7 @@ async function captureHighlightedPageState(
       `📸 [${logLabel}] Screenshot captured (with in-page highlights), size: ${screenshotResult.imageData.length} bytes`,
     );
     screenshotMs = Date.now() - screenshotStart;
-    console.log(
-      `⏱️ [HighlightTrace] background screenshot ${screenshotMs}ms`,
-    );
+    console.log(`⏱️ [HighlightTrace] background screenshot ${screenshotMs}ms`);
 
     // Apply bboxes returned from the highlight injection script
     const preCaptureData = screenshotResult.preCaptureResult;
@@ -849,7 +847,8 @@ async function captureHighlightedPageState(
       pageState,
       readinessReasons,
       _perf: {
-        scan_ms: typeof inPagePerf.scan_ms === 'number' ? inPagePerf.scan_ms : 0,
+        scan_ms:
+          typeof inPagePerf.scan_ms === 'number' ? inPagePerf.scan_ms : 0,
         scan_stats: inPagePerf.scan_stats || {},
         scan_times: inPagePerf.scan_times || {},
         pagination_ms: paginationMs,
diff --git a/extension/src/commands/highlight-detection.injected.js b/extension/src/commands/highlight-detection.injected.js
index 9581d94..4140016 100644
--- a/extension/src/commands/highlight-detection.injected.js
+++ b/extension/src/commands/highlight-detection.injected.js
@@ -2874,11 +2874,7 @@ async function runOpenBrowserHighlightDetection(config) {
   const layoutStability = evaluateReadinessSnapshot(trace);
 
   const scanStart = performance.now();
-  const scanResult = collectHighlightCandidates(
-    config,
-    trace,
-    layoutStability,
-  );
+  const scanResult = collectHighlightCandidates(config, trace, layoutStability);
   const { elements, counts } = scanResult;
   const scanMs = Math.round(performance.now() - scanStart);
 
diff --git a/extension/src/utils/collision-detection.ts b/extension/src/utils/collision-detection.ts
index da4911a..a409c64 100644
--- a/extension/src/utils/collision-detection.ts
+++ b/extension/src/utils/collision-detection.ts
@@ -572,7 +572,10 @@ function chooseLeastBlockingPlacement(
     );
     let union = fc.element.bbox;
     for (const pos of POSITION_PRIORITY) {
-      union = unionBBox(union, getLabelBBox(fc.element.bbox, pos, fc.element.id));
+      union = unionBBox(
+        union,
+        getLabelBBox(fc.element.bbox, pos, fc.element.id),
+      );
     }
     return {
       candidate: fc,