From 6492301515c2d10c68fb450e0a8c8e0cbb5a1e53 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 27 Jul 2026 21:07:57 -0700 Subject: [PATCH 1/4] Fix in-app browser annotation shortcuts --- apps/desktop/src/ipc/methods/preview.ts | 4 +- .../src/preview/AnnotationKeyboard.test.ts | 28 +++++++++ .../desktop/src/preview/AnnotationKeyboard.ts | 16 +++++ apps/desktop/src/preview/Manager.test.ts | 22 +++++++ apps/desktop/src/preview/Manager.ts | 34 ++++++++-- apps/desktop/src/preview/PickPreload.ts | 22 +++++-- apps/web/src/components/ChatView.tsx | 36 ++++++++++- .../preview/PreviewChromeRow.test.tsx | 25 ++++++++ .../components/preview/PreviewChromeRow.tsx | 5 +- .../src/components/preview/PreviewPanel.tsx | 17 ++++- .../components/preview/PreviewView.test.tsx | 47 +++++++++++++- .../src/components/preview/PreviewView.tsx | 62 ++++++++++--------- .../preview/previewUrlPresentation.test.ts | 45 -------------- .../preview/previewUrlPresentation.ts | 27 -------- packages/contracts/src/ipc.ts | 20 +++++- 15 files changed, 283 insertions(+), 127 deletions(-) create mode 100644 apps/desktop/src/preview/AnnotationKeyboard.test.ts create mode 100644 apps/desktop/src/preview/AnnotationKeyboard.ts create mode 100644 apps/web/src/components/preview/PreviewChromeRow.test.tsx delete mode 100644 apps/web/src/components/preview/previewUrlPresentation.test.ts delete mode 100644 apps/web/src/components/preview/previewUrlPresentation.ts diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 4d50ad8d665..d9980372101 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -16,7 +16,7 @@ import { DesktopPreviewSetColorSchemeInputSchema, DesktopPreviewTabInputSchema, DesktopPreviewWebviewConfigSchema, - PreviewAnnotationPayloadSchema, + PreviewAnnotationResultSchema, PreviewAutomationSnapshot, PreviewAutomationStatus, } from "@t3tools/contracts"; @@ -227,7 +227,7 @@ export const setAnnotationTheme = DesktopIpc.makeIpcMethod({ export const pickElement = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, payload: DesktopPreviewTabInputSchema, - result: Schema.NullOr(PreviewAnnotationPayloadSchema), + result: Schema.NullOr(PreviewAnnotationResultSchema), handler: Effect.fn("desktop.ipc.preview.pickElement")(function* ({ tabId }) { const manager = yield* PreviewManager.PreviewManager; return yield* manager.pickElement(tabId); diff --git a/apps/desktop/src/preview/AnnotationKeyboard.test.ts b/apps/desktop/src/preview/AnnotationKeyboard.test.ts new file mode 100644 index 00000000000..f49c1cb79f1 --- /dev/null +++ b/apps/desktop/src/preview/AnnotationKeyboard.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts"; + +const keyboardEvent = ( + overrides: Partial[0]> = {}, +) => ({ + key: "Enter", + metaKey: false, + ctrlKey: false, + shiftKey: false, + isComposing: false, + ...overrides, +}); + +describe("resolveAnnotationSubmission", () => { + it("attaches on Enter and sends on Cmd/Ctrl+Enter", () => { + expect(resolveAnnotationSubmission(keyboardEvent())).toBe("attach"); + expect(resolveAnnotationSubmission(keyboardEvent({ metaKey: true }))).toBe("send"); + expect(resolveAnnotationSubmission(keyboardEvent({ ctrlKey: true }))).toBe("send"); + }); + + it("leaves Shift+Enter and composition events available for editing", () => { + expect(resolveAnnotationSubmission(keyboardEvent({ shiftKey: true }))).toBeNull(); + expect(resolveAnnotationSubmission(keyboardEvent({ isComposing: true }))).toBeNull(); + expect(resolveAnnotationSubmission(keyboardEvent({ key: " " }))).toBeNull(); + }); +}); diff --git a/apps/desktop/src/preview/AnnotationKeyboard.ts b/apps/desktop/src/preview/AnnotationKeyboard.ts new file mode 100644 index 00000000000..6c694ccd2ed --- /dev/null +++ b/apps/desktop/src/preview/AnnotationKeyboard.ts @@ -0,0 +1,16 @@ +import type { PreviewAnnotationSubmission } from "@t3tools/contracts"; + +interface AnnotationKeyboardEvent { + readonly key: string; + readonly metaKey: boolean; + readonly ctrlKey: boolean; + readonly shiftKey: boolean; + readonly isComposing: boolean; +} + +export function resolveAnnotationSubmission( + event: AnnotationKeyboardEvent, +): PreviewAnnotationSubmission | null { + if (event.key !== "Enter" || event.shiftKey || event.isComposing) return null; + return event.metaKey || event.ctrlKey ? "send" : "attach"; +} diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index 684d6655da5..a6ef30c2742 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -36,6 +36,28 @@ describe("fitPictureInPictureContentSize", () => { }); }); +describe("isPreviewRefreshShortcut", () => { + const input = (overrides: Partial = {}) => + ({ + type: "keyDown", + key: "r", + meta: true, + control: false, + shift: false, + alt: false, + ...overrides, + }) as Electron.Input; + + it("recognizes the platform refresh chord without matching modified variants", () => { + expect(PreviewManager.isPreviewRefreshShortcut(input())).toBe(true); + expect(PreviewManager.isPreviewRefreshShortcut(input({ meta: false, control: true }))).toBe( + true, + ); + expect(PreviewManager.isPreviewRefreshShortcut(input({ shift: true }))).toBe(false); + expect(PreviewManager.isPreviewRefreshShortcut(input({ type: "keyUp" }))).toBe(false); + }); +}); + const { browserWindowConstructor, createFromPath, diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 62d400fbda6..25b625e8f2b 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -11,6 +11,7 @@ import type { DesktopPreviewPointerEvent, PreviewAnnotationPayload, PreviewAnnotationRect, + PreviewAnnotationResult, DesktopPreviewRecordingArtifact, DesktopPreviewRecordingFrame, DesktopPreviewScreenshotArtifact, @@ -406,6 +407,13 @@ const APP_FORWARDED_SHORTCUTS: ReadonlyArray<{ { key: "w", meta: true, shift: false, control: false }, ]); +export const isPreviewRefreshShortcut = (input: Electron.Input): boolean => + input.type === "keyDown" && + input.key.toLowerCase() === "r" && + (input.meta || input.control) && + !input.shift && + !input.alt; + const isPreviewInputSignal = (value: unknown): value is PreviewInputSignal => { if (typeof value !== "object" || value === null || !("kind" in value)) return false; if (value.kind === "pointer") { @@ -1365,6 +1373,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); }); const beforeInput = (event: Electron.Event, input: Electron.Input): void => { + if (isPreviewRefreshShortcut(input)) { + event.preventDefault(); + runFork( + attempt({ operation: "shortcut.refresh", tabId, webContentsId: wc.id }, () => + wc.reload(), + ).pipe(Effect.ignore), + ); + return; + } runFork(forwardShortcut(event, input)); }; yield* Scope.addFinalizer( @@ -1792,7 +1809,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const wc = yield* requireWebContents(tabId); yield* cancelPickElement(tabId); const annotationTheme = yield* Ref.get(annotationThemeRef); - return yield* Effect.callback( + return yield* Effect.callback( (resume) => { const cleanup = Effect.fn("PreviewManager.cleanupPickElement")(function* () { yield* attempt({ operation: "pickElement.cleanup", tabId, webContentsId: wc.id }, () => { @@ -1807,14 +1824,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); const settlePick = Effect.fn("PreviewManager.settlePickElement")(function* ( - payload: PreviewAnnotationPayload | null, + payload: PreviewAnnotationResult | null, ) { const active = (yield* Ref.get(pickSessionsRef)).get(tabId); if (!active || active.cancel !== cancel) return; yield* cleanup(); resume(Effect.succeed(payload)); }); - const settle = (payload: PreviewAnnotationPayload | null) => { + const settle = (payload: PreviewAnnotationResult | null) => { runFork(settlePick(payload)); }; const cancelPickSession = Effect.fn("PreviewManager.cancelPickSession")(function* () { @@ -1844,11 +1861,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return; } const cropRect = normalizeCaptureRect(args[1]); + const submission = args[2] === "send" ? "send" : "attach"; runFork( captureAnnotationScreenshot(tabId, wc, cropRect).pipe( Effect.matchEffect({ - onFailure: () => Effect.sync(() => settle(payload)), - onSuccess: (screenshot) => Effect.sync(() => settle({ ...payload, screenshot })), + onFailure: () => + Effect.sync(() => settle({ annotation: payload, submission })), + onSuccess: (screenshot) => + Effect.sync(() => + settle({ annotation: { ...payload, screenshot }, submission }), + ), }), Effect.ensuring( attempt( @@ -3586,7 +3608,7 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly pickElement: ( tabId: string, - ) => Effect.Effect; + ) => Effect.Effect; readonly cancelPickElement: (tabId: string) => Effect.Effect; readonly captureScreenshot: ( tabId: string, diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 2654b898102..51a6dea1e6c 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -11,8 +11,10 @@ import type { PreviewAnnotationRegionTarget, PreviewAnnotationStrokeTarget, PreviewAnnotationStyleChange, + PreviewAnnotationSubmission, } from "@t3tools/contracts"; +import { resolveAnnotationSubmission } from "./AnnotationKeyboard.ts"; import { previewAnnotationStyles } from "./AnnotationStyles.generated.ts"; import { ANNOTATION_CAPTURED_CHANNEL, @@ -375,6 +377,11 @@ function startAnnotation(): void { const hoverOutline = createBox(PRIMARY, PRIMARY_FILL); const marqueeBox = createBox(PRIMARY, PRIMARY_FILL); root.append(hoverOutline, marqueeBox); + root.addEventListener("keydown", (event) => { + // Let the focused annotation control handle the key, then keep it away + // from page-level shortcuts such as a video's Space play/pause binding. + event.stopPropagation(); + }); const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); svg.setAttribute(OVERLAY_ATTRIBUTE, ""); @@ -426,7 +433,7 @@ function startAnnotation(): void { "hidden h-8 w-6 shrink-0 cursor-grab select-none border-0 bg-transparent p-0 font-sans text-lg font-bold leading-5 text-muted-foreground"; composerRow.appendChild(dragHandle); - const submit = createButton("Attach", "Attach annotation and screenshot"); + const submit = createButton("Attach", "Attach annotation and screenshot (Enter)"); submit.className += " h-8 shrink-0 border-primary bg-primary px-3 text-primary-foreground shadow-sm hover:bg-primary/90"; composerRow.appendChild(submit); @@ -1182,7 +1189,7 @@ function startAnnotation(): void { refreshToolButtons(); }; - submit.addEventListener("click", () => { + const submitAnnotation = (submission: PreviewAnnotationSubmission): void => { if (pendingCapture || (selected.size === 0 && regions.length === 0 && strokes.length === 0)) return; pendingCapture = true; @@ -1223,13 +1230,16 @@ function startAnnotation(): void { ...regions.map((region) => region.rect), ...strokes.map((stroke) => stroke.bounds), ]); - ipcRenderer.send(ELEMENT_PICKED_CHANNEL, annotation, screenshotRect); + ipcRenderer.send(ELEMENT_PICKED_CHANNEL, annotation, screenshotRect, submission); }); - }); + }; + submit.addEventListener("click", () => submitAnnotation("attach")); comment.addEventListener("keydown", (event) => { - if (event.key !== "Enter" || !(event.metaKey || event.ctrlKey)) return; + const submission = resolveAnnotationSubmission(event); + if (!submission) return; event.preventDefault(); - submit.click(); + event.stopPropagation(); + submitAnnotation(submission); }); window.addEventListener("pointermove", onPointerMove, { capture: true, passive: false }); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index c1c49fec8fb..0a7bd5f2f9f 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -8,6 +8,7 @@ import { type ProjectScript, type ProjectId, type ProviderApprovalDecision, + type PreviewAnnotationPayload, ProviderInstanceId, type ServerProvider, type ResolvedKeybindingsConfig, @@ -4561,7 +4562,13 @@ function ChatViewContent(props: ChatViewProps) { ], ); - const onSend = async (e?: { preventDefault: () => void }) => { + const onSend = async ( + e?: { preventDefault: () => void }, + directAnnotation?: { + annotation: PreviewAnnotationPayload; + image: ComposerImageAttachment | null; + }, + ) => { e?.preventDefault(); if ( !activeThread || @@ -4579,10 +4586,10 @@ function ChatViewContent(props: ChatViewProps) { const sendCtx = composerRef.current?.getSendContext(); if (!sendCtx?.providerAvailable) return; const { - images: composerImages, + images: sendContextImages, terminalContexts: composerTerminalContexts, elementContexts: composerElementContexts, - previewAnnotations: composerPreviewAnnotations, + previewAnnotations: sendContextPreviewAnnotations, reviewComments: composerReviewComments, selectedProvider: ctxSelectedProvider, selectedModel: ctxSelectedModel, @@ -4590,6 +4597,26 @@ function ChatViewContent(props: ChatViewProps) { selectedPromptEffort: ctxSelectedPromptEffort, selectedModelSelection: ctxSelectedModelSelection, } = sendCtx; + const composerImages = + directAnnotation?.image && + !sendContextImages.some((image) => image.id === directAnnotation.image?.id) + ? [...sendContextImages, directAnnotation.image] + : sendContextImages; + const composerPreviewAnnotations = + directAnnotation && + !sendContextPreviewAnnotations.some( + (annotation) => annotation.id === directAnnotation.annotation.id, + ) + ? [ + ...sendContextPreviewAnnotations, + { + ...directAnnotation.annotation, + screenshot: directAnnotation.annotation.screenshot + ? { ...directAnnotation.annotation.screenshot, dataUrl: "" } + : null, + }, + ] + : sendContextPreviewAnnotations; const promptForSend = promptRef.current; const { trimmedPrompt: trimmed, @@ -5667,6 +5694,9 @@ function ChatViewContent(props: ChatViewProps) { tabId={activeRightPanelSurface.resourceId} configuredUrls={configuredPreviewUrls} visible + onSendAnnotation={(annotation, image) => { + void onSend(undefined, { annotation, image }); + }} /> ) : activeRightPanelSurface?.kind === "terminal" ? ( diff --git a/apps/web/src/components/preview/PreviewChromeRow.test.tsx b/apps/web/src/components/preview/PreviewChromeRow.test.tsx new file mode 100644 index 00000000000..77e13fb421c --- /dev/null +++ b/apps/web/src/components/preview/PreviewChromeRow.test.tsx @@ -0,0 +1,25 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vite-plus/test"; + +import { PreviewChromeRow } from "./PreviewChromeRow"; + +describe("PreviewChromeRow", () => { + it("shows the complete URL while the address bar is not focused", () => { + const markup = renderToStaticMarkup( + , + ); + + expect(markup).toContain('value="https://example.com/dashboard?mode=edit&tab=1#notes"'); + }); +}); diff --git a/apps/web/src/components/preview/PreviewChromeRow.tsx b/apps/web/src/components/preview/PreviewChromeRow.tsx index 1f4eb95d62b..958b30a4797 100644 --- a/apps/web/src/components/preview/PreviewChromeRow.tsx +++ b/apps/web/src/components/preview/PreviewChromeRow.tsx @@ -23,7 +23,6 @@ import { cn } from "~/lib/utils"; interface Props { url: string; - displayUrl?: string | undefined; loading: boolean; loadProgress: number; canGoBack: boolean; @@ -65,7 +64,6 @@ const NOOP = () => {}; export function PreviewChromeRow({ url, - displayUrl, loading, loadProgress, canGoBack, @@ -172,7 +170,7 @@ export function PreviewChromeRow({ render={ } /> - {!inputFocused && displayUrl ? {url} : null} {onOpenInBrowser && !inputFocused ? ( | undefined; visible: boolean; + onSendAnnotation?: ( + annotation: PreviewAnnotationPayload, + image: ComposerImageAttachment | null, + ) => void; } -export function PreviewPanel({ mode, threadRef, tabId, configuredUrls, visible }: Props) { +export function PreviewPanel({ + mode, + threadRef, + tabId, + configuredUrls, + visible, + onSendAnnotation, +}: Props) { if (!isPreviewSupportedInRuntime()) { return ( @@ -35,6 +47,7 @@ export function PreviewPanel({ mode, threadRef, tabId, configuredUrls, visible } {...(tabId !== undefined ? { tabId } : {})} configuredUrls={configuredUrls} visible={visible} + {...(onSendAnnotation ? { onSendAnnotation } : {})} /> ); diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 576c37d77b7..22b174d0307 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -17,6 +17,10 @@ const mocks = vi.hoisted(() => ({ closeRightPanel: vi.fn(), openPictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), closePictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), + pickElement: vi.fn(), + addPreviewAnnotation: vi.fn(), + addImage: vi.fn(), + toggleAnnotation: null as (() => void) | null, pictureInPicture: false, showEmptyState: false, })); @@ -28,7 +32,11 @@ vi.mock("~/state/session", () => ({ vi.mock("~/composerDraftStore", () => ({ useComposerDraftStore: ( select: (store: { addPreviewAnnotation: () => void; addImage: () => void }) => unknown, - ) => select({ addPreviewAnnotation: vi.fn(), addImage: vi.fn() }), + ) => + select({ + addPreviewAnnotation: mocks.addPreviewAnnotation, + addImage: mocks.addImage, + }), })); vi.mock("~/lib/previewAnnotation", () => ({ @@ -144,6 +152,7 @@ vi.mock("~/components/ui/toast", () => ({ vi.mock("./previewBridge", () => ({ previewBridge: { navigate: mocks.navigate, + pickElement: mocks.pickElement, pictureInPicture: { open: mocks.openPictureInPicture, close: mocks.closePictureInPicture, @@ -154,6 +163,7 @@ vi.mock("./previewBridge", () => ({ vi.mock("./PreviewChromeRow", () => ({ PreviewChromeRow: (props: { onSubmit: (url: string) => void; + onPickElement?: () => void; onPictureInPicture?: () => void; pictureInPicture?: boolean; trailingActions?: { @@ -161,6 +171,7 @@ vi.mock("./PreviewChromeRow", () => ({ }; }) => { mocks.submittedUrl = props.onSubmit; + mocks.toggleAnnotation = props.onPickElement ?? null; mocks.togglePictureInPicture = props.onPictureInPicture ?? null; mocks.toggleNativePictureInPicture = props.trailingActions?.props.onNativePictureInPicture ?? null; @@ -213,6 +224,10 @@ describe("PreviewView navigation", () => { mocks.closeRightPanel.mockClear(); mocks.openPictureInPicture.mockClear(); mocks.closePictureInPicture.mockClear(); + mocks.pickElement.mockReset(); + mocks.addPreviewAnnotation.mockClear(); + mocks.addImage.mockClear(); + mocks.toggleAnnotation = null; mocks.pictureInPicture = false; mocks.showEmptyState = false; }); @@ -327,4 +342,34 @@ describe("PreviewView navigation", () => { expect(mocks.closePictureInPicture).toHaveBeenCalledWith(TEST_RUNTIME_TAB_ID), ); }); + + it("forwards Cmd/Ctrl+Enter annotations to the composer send path", async () => { + const annotation = { + id: "annotation-1", + pageUrl: "https://example.com/dashboard", + pageTitle: "Dashboard", + comment: "Tighten this spacing", + elements: [], + regions: [], + strokes: [], + styleChanges: [], + screenshot: null, + createdAt: "2026-07-27T00:00:00.000Z", + }; + const onSendAnnotation = vi.fn(); + mocks.pickElement.mockResolvedValue({ annotation, submission: "send" }); + + renderToStaticMarkup( + , + ); + mocks.toggleAnnotation?.(); + + await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null)); + expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, annotation); + }); }); diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index a3e23c2c41c..291ad8c9c06 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -4,13 +4,14 @@ import { scopedThreadKey } from "@t3tools/client-runtime/environment"; import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import { FILL_PREVIEW_VIEWPORT, + type PreviewAnnotationPayload, type PreviewViewportSetting, type ScopedThreadRef, } from "@t3tools/contracts"; import { normalizePreviewUrl } from "@t3tools/shared/preview"; import { useCallback, useEffect, useRef, useState } from "react"; -import { useComposerDraftStore } from "~/composerDraftStore"; +import { type ComposerImageAttachment, useComposerDraftStore } from "~/composerDraftStore"; import { previewAnnotationScreenshotFile } from "~/lib/previewAnnotation"; import { ensureLocalApi } from "~/localApi"; import { @@ -19,7 +20,6 @@ import { useThreadPreviewState, } from "~/previewStateStore"; import { resolveDiscoveredServerUrl } from "~/browser/browserTargetResolver"; -import { useEnvironment, useEnvironmentHttpBaseUrl } from "~/state/environments"; import { previewEnvironment } from "~/state/preview"; import { useAtomCommand } from "~/state/use-atom-command"; import { selectThreadPreviewMiniPlayer, usePreviewMiniPlayerStore } from "~/previewMiniPlayerStore"; @@ -29,7 +29,6 @@ import { previewBridge } from "./previewBridge"; import { subscribePreviewAction } from "./previewActionBus"; import { openPreviewSession } from "./openPreviewSession"; import { PreviewChromeRow } from "./PreviewChromeRow"; -import { formatPreviewUrl } from "./previewUrlPresentation"; import { PreviewEmptyState } from "./PreviewEmptyState"; import { PreviewMoreMenu } from "./PreviewMoreMenu"; import { @@ -60,6 +59,10 @@ interface Props { tabId?: string | null; configuredUrls?: ReadonlyArray | undefined; visible: boolean; + onSendAnnotation?: ( + annotation: PreviewAnnotationPayload, + image: ComposerImageAttachment | null, + ) => void; } const localApi = typeof window === "undefined" ? null : ensureLocalApi(); @@ -68,7 +71,13 @@ const localApi = typeof window === "undefined" ? null : ensureLocalApi(); * Single-tab preview surface: chrome row on top, one webview below, empty * state when no session exists for the thread. */ -export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, visible }: Props) { +export function PreviewView({ + threadRef, + tabId: requestedTabId, + configuredUrls, + visible, + onSendAnnotation, +}: Props) { const [focusUrlNonce, setFocusUrlNonce] = useState(undefined); const [pickActive, setPickActive] = useState(false); const activeRecordingTabIds = useActiveBrowserRecordingTabIds(); @@ -80,8 +89,6 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, ); const addPreviewAnnotation = useComposerDraftStore((store) => store.addPreviewAnnotation); const addImage = useComposerDraftStore((store) => store.addImage); - const environment = useEnvironment(threadRef.environmentId); - const environmentHttpBaseUrl = useEnvironmentHttpBaseUrl(threadRef.environmentId); const open = useAtomCommand(previewEnvironment.open); const resize = useAtomCommand(previewEnvironment.resize, "preview viewport resize"); @@ -116,14 +123,6 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, const showEmptyState = shouldShowPreviewEmptyState(snapshot); const controller = desktopOverlay?.controller ?? "none"; const loadProgress = useLoadingProgress(loading); - const displayUrl = - url && environment && environmentHttpBaseUrl - ? (formatPreviewUrl({ - url, - environmentLabel: environment.label, - environmentHttpBaseUrl, - }) ?? undefined) - : undefined; const viewport = snapshot?.viewport ?? FILL_PREVIEW_VIEWPORT; const panelRect = useBrowserSurfaceStore((state) => runtimeTabId ? (state.byTabId[runtimeTabId]?.rect ?? null) : null, @@ -523,20 +522,28 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, setPickActive(true); void (async () => { try { - const annotation = await previewBridge.pickElement(runtimeTabId); - if (!annotation) return; + const result = await previewBridge.pickElement(runtimeTabId); + if (!result) return; + const { annotation, submission } = result; addPreviewAnnotation(threadRef, annotation); const screenshotFile = await previewAnnotationScreenshotFile(annotation); - if (screenshotFile && annotation.screenshot) { - addImage(threadRef, { - type: "image", - id: annotation.id, - name: screenshotFile.name, - mimeType: screenshotFile.type, - sizeBytes: screenshotFile.size, - previewUrl: annotation.screenshot.dataUrl, - file: screenshotFile, - }); + const image = + screenshotFile && annotation.screenshot + ? ({ + type: "image", + id: annotation.id, + name: screenshotFile.name, + mimeType: screenshotFile.type, + sizeBytes: screenshotFile.size, + previewUrl: annotation.screenshot.dataUrl, + file: screenshotFile, + } satisfies ComposerImageAttachment) + : null; + if (image) { + addImage(threadRef, image); + } + if (submission === "send") { + onSendAnnotation?.(annotation, image); } } catch { // Picker failed (e.g. webview navigated). Treat as silent cancel. @@ -561,7 +568,7 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, } } })(); - }, [addImage, addPreviewAnnotation, runtimeTabId, threadRef]); + }, [addImage, addPreviewAnnotation, onSendAnnotation, runtimeTabId, threadRef]); // If the active tab changes mid-pick (close, thread switch, hot restart), // tell main to tear down the in-flight session AND reset our local toggle @@ -611,7 +618,6 @@ export function PreviewView({ threadRef, tabId: requestedTabId, configuredUrls, > { - it("formats signed asset URLs with the environment label and decoded filename", () => { - expect( - formatPreviewUrl({ - url: "http://127.0.0.1:3773/api/assets/token/architecture%20brief.pdf", - environmentLabel: "Local environment", - environmentHttpBaseUrl: "http://127.0.0.1:3773", - }), - ).toBe("Local environment · architecture brief.pdf"); - }); - - it("does not alias assets from another origin", () => { - expect( - formatPreviewUrl({ - url: "https://example.com/api/assets/token/report.pdf", - environmentLabel: "Local environment", - environmentHttpBaseUrl: "http://127.0.0.1:3773", - }), - ).toBe("example.com"); - }); - - it("formats regular preview URLs as their exact host", () => { - expect( - formatPreviewUrl({ - url: "http://127.0.0.1:5173/dashboard", - environmentLabel: "Local environment", - environmentHttpBaseUrl: "http://127.0.0.1:3773", - }), - ).toBe("127.0.0.1:5173"); - }); - - it("does not compact non-http URLs", () => { - expect( - formatPreviewUrl({ - url: "file:///tmp/report.pdf", - environmentLabel: "Local environment", - environmentHttpBaseUrl: "http://127.0.0.1:3773", - }), - ).toBeNull(); - }); -}); diff --git a/apps/web/src/components/preview/previewUrlPresentation.ts b/apps/web/src/components/preview/previewUrlPresentation.ts deleted file mode 100644 index 0ae3c1900aa..00000000000 --- a/apps/web/src/components/preview/previewUrlPresentation.ts +++ /dev/null @@ -1,27 +0,0 @@ -interface PreviewUrlPresentationInput { - readonly url: string; - readonly environmentLabel: string; - readonly environmentHttpBaseUrl: string; -} - -export function formatPreviewUrl(input: PreviewUrlPresentationInput): string | null { - try { - const url = new URL(input.url); - const environmentUrl = new URL(input.environmentHttpBaseUrl); - if (url.origin === environmentUrl.origin && url.pathname.startsWith("/api/assets/")) { - const encodedFileName = url.pathname.split("/").at(-1); - if (!encodedFileName) { - return null; - } - const fileName = decodeURIComponent(encodedFileName); - if (!fileName || fileName === "." || fileName === "..") { - return null; - } - return `${input.environmentLabel} · ${fileName}`; - } - - return url.protocol === "http:" || url.protocol === "https:" ? url.host : null; - } catch { - return null; - } -} diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index a92eb972d67..ffd2276cc1a 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -891,6 +891,19 @@ export const PreviewAnnotationPayloadSchema: Schema.Codec = + Schema.Literals(["attach", "send"]); + +export interface PreviewAnnotationResult { + annotation: PreviewAnnotationPayload; + submission: PreviewAnnotationSubmission; +} +export const PreviewAnnotationResultSchema: Schema.Codec = Schema.Struct({ + annotation: PreviewAnnotationPayloadSchema, + submission: PreviewAnnotationSubmissionSchema, +}); + export const DesktopPreviewTabInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, }); @@ -1057,10 +1070,11 @@ export interface DesktopPreviewBridge { setAnnotationTheme: (theme: DesktopPreviewAnnotationTheme) => Promise; /** * Activate the in-page element picker for the given tab. Resolves with - * the picked payload, or `null` when the user cancels (Escape / nav). The - * promise rejects if the picker can't be activated (no webview, etc.). + * the picked annotation and its attach/send intent, or `null` when the + * user cancels (Escape / nav). The promise rejects if the picker can't be + * activated (no webview, etc.). */ - pickElement: (tabId: string) => Promise; + pickElement: (tabId: string) => Promise; /** Cancel an in-flight preview annotation session. */ cancelPickElement: (tabId: string) => Promise; captureScreenshot: (tabId: string) => Promise; From 3a3b0e611cc8b1b0d32d9c8861e723bc655dfc75 Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 27 Jul 2026 21:22:33 -0700 Subject: [PATCH 2/4] Address in-app browser review feedback --- apps/desktop/src/ipc/methods/preview.ts | 4 +- apps/desktop/src/preview/Manager.ts | 17 ++++---- apps/desktop/src/preview/PickPreload.ts | 25 ++++++------ apps/web/src/components/ChatView.tsx | 23 ++++++++++- .../components/preview/PreviewView.test.tsx | 40 ++++++++++++++++++- .../src/components/preview/PreviewView.tsx | 8 +++- packages/contracts/src/ipc.ts | 13 +++--- 7 files changed, 96 insertions(+), 34 deletions(-) diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index d9980372101..febdefa9825 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -16,7 +16,7 @@ import { DesktopPreviewSetColorSchemeInputSchema, DesktopPreviewTabInputSchema, DesktopPreviewWebviewConfigSchema, - PreviewAnnotationResultSchema, + PreviewAnnotationSubmissionResultSchema, PreviewAutomationSnapshot, PreviewAutomationStatus, } from "@t3tools/contracts"; @@ -227,7 +227,7 @@ export const setAnnotationTheme = DesktopIpc.makeIpcMethod({ export const pickElement = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_PICK_ELEMENT_CHANNEL, payload: DesktopPreviewTabInputSchema, - result: Schema.NullOr(PreviewAnnotationResultSchema), + result: Schema.NullOr(PreviewAnnotationSubmissionResultSchema), handler: Effect.fn("desktop.ipc.preview.pickElement")(function* ({ tabId }) { const manager = yield* PreviewManager.PreviewManager; return yield* manager.pickElement(tabId); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 25b625e8f2b..169fe2992dc 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -11,7 +11,7 @@ import type { DesktopPreviewPointerEvent, PreviewAnnotationPayload, PreviewAnnotationRect, - PreviewAnnotationResult, + PreviewAnnotationSubmissionResult, DesktopPreviewRecordingArtifact, DesktopPreviewRecordingFrame, DesktopPreviewScreenshotArtifact, @@ -1809,7 +1809,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const wc = yield* requireWebContents(tabId); yield* cancelPickElement(tabId); const annotationTheme = yield* Ref.get(annotationThemeRef); - return yield* Effect.callback( + return yield* Effect.callback( (resume) => { const cleanup = Effect.fn("PreviewManager.cleanupPickElement")(function* () { yield* attempt({ operation: "pickElement.cleanup", tabId, webContentsId: wc.id }, () => { @@ -1824,14 +1824,14 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); const settlePick = Effect.fn("PreviewManager.settlePickElement")(function* ( - payload: PreviewAnnotationResult | null, + payload: PreviewAnnotationSubmissionResult | null, ) { const active = (yield* Ref.get(pickSessionsRef)).get(tabId); if (!active || active.cancel !== cancel) return; yield* cleanup(); resume(Effect.succeed(payload)); }); - const settle = (payload: PreviewAnnotationResult | null) => { + const settle = (payload: PreviewAnnotationSubmissionResult | null) => { runFork(settlePick(payload)); }; const cancelPickSession = Effect.fn("PreviewManager.cancelPickSession")(function* () { @@ -1865,12 +1865,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function runFork( captureAnnotationScreenshot(tabId, wc, cropRect).pipe( Effect.matchEffect({ - onFailure: () => - Effect.sync(() => settle({ annotation: payload, submission })), + onFailure: () => Effect.sync(() => settle({ annotation: payload, submission })), onSuccess: (screenshot) => - Effect.sync(() => - settle({ annotation: { ...payload, screenshot }, submission }), - ), + Effect.sync(() => settle({ annotation: { ...payload, screenshot }, submission })), }), Effect.ensuring( attempt( @@ -3608,7 +3605,7 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly pickElement: ( tabId: string, - ) => Effect.Effect; + ) => Effect.Effect; readonly cancelPickElement: (tabId: string) => Effect.Effect; readonly captureScreenshot: ( tabId: string, diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 51a6dea1e6c..9c795d66c14 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -377,11 +377,6 @@ function startAnnotation(): void { const hoverOutline = createBox(PRIMARY, PRIMARY_FILL); const marqueeBox = createBox(PRIMARY, PRIMARY_FILL); root.append(hoverOutline, marqueeBox); - root.addEventListener("keydown", (event) => { - // Let the focused annotation control handle the key, then keep it away - // from page-level shortcuts such as a video's Space play/pause binding. - event.stopPropagation(); - }); const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); svg.setAttribute(OVERLAY_ATTRIBUTE, ""); @@ -1234,13 +1229,19 @@ function startAnnotation(): void { }); }; submit.addEventListener("click", () => submitAnnotation("attach")); - comment.addEventListener("keydown", (event) => { - const submission = resolveAnnotationSubmission(event); - if (!submission) return; - event.preventDefault(); - event.stopPropagation(); - submitAnnotation(submission); - }); + root.addEventListener( + "keydown", + (event) => { + const submission = event.target === comment ? resolveAnnotationSubmission(event) : null; + // Isolate annotation controls before the event reaches bubble-phase + // listeners installed by the inspected page. + event.stopImmediatePropagation(); + if (!submission) return; + event.preventDefault(); + submitAnnotation(submission); + }, + { capture: true }, + ); window.addEventListener("pointermove", onPointerMove, { capture: true, passive: false }); window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: false }); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 0a7bd5f2f9f..eedaf87bb72 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4570,6 +4570,16 @@ function ChatViewContent(props: ChatViewProps) { }, ) => { e?.preventDefault(); + const notifyDirectAnnotationAttached = () => { + if (!directAnnotation) return; + toastManager.add( + stackedThreadToast({ + type: "info", + title: "Annotation attached to draft", + description: "Sending is unavailable right now. Finish the current action, then send.", + }), + ); + }; if ( !activeThread || isSendBusy || @@ -4577,14 +4587,23 @@ function ChatViewContent(props: ChatViewProps) { threadDetailLoading || activeEnvironmentUnavailable || sendInFlightRef.current - ) + ) { + notifyDirectAnnotationAttached(); return; + } if (activePendingProgress) { + if (directAnnotation) { + notifyDirectAnnotationAttached(); + return; + } onAdvanceActivePendingUserInput(); return; } const sendCtx = composerRef.current?.getSendContext(); - if (!sendCtx?.providerAvailable) return; + if (!sendCtx?.providerAvailable) { + notifyDirectAnnotationAttached(); + return; + } const { images: sendContextImages, terminalContexts: composerTerminalContexts, diff --git a/apps/web/src/components/preview/PreviewView.test.tsx b/apps/web/src/components/preview/PreviewView.test.tsx index 22b174d0307..4121b72602f 100644 --- a/apps/web/src/components/preview/PreviewView.test.tsx +++ b/apps/web/src/components/preview/PreviewView.test.tsx @@ -18,6 +18,7 @@ const mocks = vi.hoisted(() => ({ openPictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), closePictureInPicture: vi.fn(async (_tabId: string): Promise => undefined), pickElement: vi.fn(), + previewAnnotationScreenshotFile: vi.fn(), addPreviewAnnotation: vi.fn(), addImage: vi.fn(), toggleAnnotation: null as (() => void) | null, @@ -40,7 +41,7 @@ vi.mock("~/composerDraftStore", () => ({ })); vi.mock("~/lib/previewAnnotation", () => ({ - previewAnnotationScreenshotFile: vi.fn(), + previewAnnotationScreenshotFile: mocks.previewAnnotationScreenshotFile, })); vi.mock("~/localApi", () => ({ @@ -225,6 +226,7 @@ describe("PreviewView navigation", () => { mocks.openPictureInPicture.mockClear(); mocks.closePictureInPicture.mockClear(); mocks.pickElement.mockReset(); + mocks.previewAnnotationScreenshotFile.mockReset(); mocks.addPreviewAnnotation.mockClear(); mocks.addImage.mockClear(); mocks.toggleAnnotation = null; @@ -372,4 +374,40 @@ describe("PreviewView navigation", () => { await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null)); expect(mocks.addPreviewAnnotation).toHaveBeenCalledWith(TEST_THREAD_REF, annotation); }); + + it("still sends when screenshot attachment conversion fails", async () => { + const annotation = { + id: "annotation-2", + pageUrl: "https://example.com/dashboard", + pageTitle: "Dashboard", + comment: "Tighten this spacing", + elements: [], + regions: [], + strokes: [], + styleChanges: [], + screenshot: { + dataUrl: "data:image/png;base64,c2NyZWVuc2hvdA==", + width: 10, + height: 10, + cropRect: { x: 0, y: 0, width: 10, height: 10 }, + }, + createdAt: "2026-07-27T00:00:00.000Z", + }; + const onSendAnnotation = vi.fn(); + mocks.pickElement.mockResolvedValue({ annotation, submission: "send" }); + mocks.previewAnnotationScreenshotFile.mockRejectedValue(new Error("conversion failed")); + + renderToStaticMarkup( + , + ); + mocks.toggleAnnotation?.(); + + await vi.waitFor(() => expect(onSendAnnotation).toHaveBeenCalledWith(annotation, null)); + expect(mocks.addImage).not.toHaveBeenCalled(); + }); }); diff --git a/apps/web/src/components/preview/PreviewView.tsx b/apps/web/src/components/preview/PreviewView.tsx index 291ad8c9c06..a2435627c62 100644 --- a/apps/web/src/components/preview/PreviewView.tsx +++ b/apps/web/src/components/preview/PreviewView.tsx @@ -526,7 +526,13 @@ export function PreviewView({ if (!result) return; const { annotation, submission } = result; addPreviewAnnotation(threadRef, annotation); - const screenshotFile = await previewAnnotationScreenshotFile(annotation); + let screenshotFile: File | null = null; + try { + screenshotFile = await previewAnnotationScreenshotFile(annotation); + } catch { + // The structured annotation is still sendable when converting its + // optional screenshot into a composer attachment fails. + } const image = screenshotFile && annotation.screenshot ? ({ diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index ffd2276cc1a..49689b14c80 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -895,14 +895,15 @@ export type PreviewAnnotationSubmission = "attach" | "send"; export const PreviewAnnotationSubmissionSchema: Schema.Codec = Schema.Literals(["attach", "send"]); -export interface PreviewAnnotationResult { +export interface PreviewAnnotationSubmissionResult { annotation: PreviewAnnotationPayload; submission: PreviewAnnotationSubmission; } -export const PreviewAnnotationResultSchema: Schema.Codec = Schema.Struct({ - annotation: PreviewAnnotationPayloadSchema, - submission: PreviewAnnotationSubmissionSchema, -}); +export const PreviewAnnotationSubmissionResultSchema: Schema.Codec = + Schema.Struct({ + annotation: PreviewAnnotationPayloadSchema, + submission: PreviewAnnotationSubmissionSchema, + }); export const DesktopPreviewTabInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, @@ -1074,7 +1075,7 @@ export interface DesktopPreviewBridge { * user cancels (Escape / nav). The promise rejects if the picker can't be * activated (no webview, etc.). */ - pickElement: (tabId: string) => Promise; + pickElement: (tabId: string) => Promise; /** Cancel an in-flight preview annotation session. */ cancelPickElement: (tabId: string) => Promise; captureScreenshot: (tabId: string) => Promise; From f7d02b1dacc9fc84241fcf3362b0bfb6e098102a Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Mon, 27 Jul 2026 21:27:37 -0700 Subject: [PATCH 3/4] Preserve annotation editor keyboard input --- apps/desktop/src/preview/PickPreload.ts | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src/preview/PickPreload.ts b/apps/desktop/src/preview/PickPreload.ts index 9c795d66c14..d03673400ab 100644 --- a/apps/desktop/src/preview/PickPreload.ts +++ b/apps/desktop/src/preview/PickPreload.ts @@ -1229,19 +1229,15 @@ function startAnnotation(): void { }); }; submit.addEventListener("click", () => submitAnnotation("attach")); - root.addEventListener( - "keydown", - (event) => { - const submission = event.target === comment ? resolveAnnotationSubmission(event) : null; - // Isolate annotation controls before the event reaches bubble-phase - // listeners installed by the inspected page. - event.stopImmediatePropagation(); - if (!submission) return; - event.preventDefault(); - submitAnnotation(submission); - }, - { capture: true }, - ); + root.addEventListener("keydown", (event) => { + const submission = event.target === comment ? resolveAnnotationSubmission(event) : null; + // Keep this in the bubble phase so editor inputs receive the event before + // it is isolated from listeners installed by the inspected page. + event.stopImmediatePropagation(); + if (!submission) return; + event.preventDefault(); + submitAnnotation(submission); + }); window.addEventListener("pointermove", onPointerMove, { capture: true, passive: false }); window.addEventListener("pointerdown", onPointerDown, { capture: true, passive: false }); From 1cec8c1e6116c4bdff993c94eacee5336005ef6f Mon Sep 17 00:00:00 2001 From: Theo Browne Date: Tue, 4 Aug 2026 14:22:27 -0700 Subject: [PATCH 4/4] fix(web): preserve direct annotation sends during plan follow-up --- apps/web/src/components/ChatView.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index eedaf87bb72..5652901d56a 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -4651,7 +4651,7 @@ function ChatViewContent(props: ChatViewProps) { composerPreviewAnnotations.length + composerReviewComments.length, }); - if (showPlanFollowUpPrompt && activeProposedPlan) { + if (!directAnnotation && showPlanFollowUpPrompt && activeProposedPlan) { const followUp = resolvePlanFollowUpSubmission({ draftText: trimmed, planMarkdown: activeProposedPlan.planMarkdown,