diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index b89521a932d..e92ecd497e3 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -64,7 +64,7 @@ import { } from "../../promptStashStore"; import { ComposerStashBadge } from "./ComposerStashBadge"; import { ComposerStashMenu } from "./ComposerStashMenu"; -import { compressImageForStash } from "../../lib/stashImageCompression"; +import { compressImageForStash, compressImageToByteLimit } from "../../lib/imageCompression"; import { isCommandPaletteOpen } from "../../commandPaletteBus"; import { getTerminalFocusOwner } from "../../lib/terminalFocus"; import { resolveShortcutCommand } from "../../keybindings"; @@ -206,8 +206,6 @@ import { searchProviderSkills } from "../../providerSkillSearch"; import { useMediaQuery } from "../../hooks/useMediaQuery"; import type { ReviewCommentContext } from "../../reviewCommentContext"; -const IMAGE_SIZE_LIMIT_LABEL = `${Math.round(PROVIDER_SEND_TURN_MAX_IMAGE_BYTES / (1024 * 1024))}MB`; - const runtimeModeConfig: Record< RuntimeMode, { label: string; description: string; icon: LucideIcon } @@ -1006,6 +1004,13 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) * thread) can still be stashed while an earlier encode is running. */ const stashInFlightRef = useRef>(new Set()); + /** + * Count of pasted images still being compressed, per thread. Reserved + * against the attachment limit so concurrent pastes can't overshoot it, + * and checked by `submitComposer` so a send can't race an image into the + * next draft. + */ + const pendingImageCompressionsRef = useRef>(new Map()); // ------------------------------------------------------------------ // Derived: composer send state @@ -1821,12 +1826,26 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) event?.preventDefault(); return; } + // A send while a pasted image is still compressing would strand that + // image: the turn snapshot wouldn't include it, and it would surface + // in the *next* draft instead. Only oversized images hit this — small + // files clear the pending counter within a microtask. + if (activeThreadId && (pendingImageCompressionsRef.current.get(activeThreadId) ?? 0) > 0) { + event?.preventDefault(); + toastManager.add({ + type: "info", + title: "Still compressing a pasted image.", + description: "Send again once its thumbnail appears.", + }); + return; + } onSend(event); if (shouldBlurMobileComposerOnSubmit()) { blurMobileComposerAfterSend(); } }, [ + activeThreadId, blurMobileComposerAfterSend, isSendDisabled, noProviderAvailable, @@ -2273,7 +2292,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) // ------------------------------------------------------------------ // Callbacks: images // ------------------------------------------------------------------ - const addComposerImages = (files: File[]) => { + const addComposerImages = async (files: File[]) => { if (!activeThreadId || files.length === 0) return; if (pendingUserInputs.length > 0) { toastManager.add({ @@ -2282,40 +2301,81 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) }); return; } - const nextImages: ComposerImageAttachment[] = []; - let nextImageCount = composerImagesRef.current.length; + // Captured before the awaits below: the user may switch threads while a + // large image is being compressed, and the attachments and errors belong + // to the thread the paste happened in. + const threadId = activeThreadId; + + // Validation happens synchronously so concurrent pastes see each other: + // accepted files reserve their attachment slots (via the pending counter) + // before the first await, keeping the total under the limit. + const pendingCount = pendingImageCompressionsRef.current.get(threadId) ?? 0; + let reservedCount = composerImagesRef.current.length + pendingCount; + const acceptedFiles: File[] = []; let error: string | null = null; for (const file of files) { if (!file.type.startsWith("image/")) { error = `Unsupported file type for '${file.name}'. Please attach image files only.`; continue; } - if (file.size > PROVIDER_SEND_TURN_MAX_IMAGE_BYTES) { - error = `'${file.name}' exceeds the ${IMAGE_SIZE_LIMIT_LABEL} attachment limit.`; - continue; - } - if (nextImageCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { + if (reservedCount >= PROVIDER_SEND_TURN_MAX_ATTACHMENTS) { error = `You can attach up to ${PROVIDER_SEND_TURN_MAX_ATTACHMENTS} images per message.`; break; } - const previewUrl = URL.createObjectURL(file); - nextImages.push({ - type: "image", - id: randomUUID(), - name: file.name || "image", - mimeType: file.type, - sizeBytes: file.size, - previewUrl, - file, - }); - nextImageCount += 1; + acceptedFiles.push(file); + reservedCount += 1; } - if (nextImages.length === 1 && nextImages[0]) { - addComposerImage(nextImages[0]); - } else if (nextImages.length > 1) { - addComposerImagesToDraft(nextImages); + setThreadError(threadId, error); + if (acceptedFiles.length === 0) return; + + pendingImageCompressionsRef.current.set(threadId, pendingCount + acceptedFiles.length); + try { + const nextImages: ComposerImageAttachment[] = []; + let compressionError: string | null = null; + for (const file of acceptedFiles) { + // Images over the wire cap are downscaled to fit rather than + // refused; files already within it pass through byte-for-byte. + const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); + if (!compressed.ok) { + compressionError = + compressed.reason === "unreadable" + ? `'${file.name}' could not be read as an image.` + : `'${file.name}' is too large to attach, even after compression.`; + continue; + } + const attachmentFile = compressed.file; + const previewUrl = URL.createObjectURL(attachmentFile); + nextImages.push({ + type: "image", + id: randomUUID(), + name: attachmentFile.name || "image", + mimeType: attachmentFile.type, + sizeBytes: attachmentFile.size, + previewUrl, + file: attachmentFile, + }); + } + if (nextImages.length === 1 && nextImages[0]) { + addComposerImage(nextImages[0]); + } else if (nextImages.length > 1) { + addComposerImagesToDraft(nextImages); + } + // Only failures are reported here. Success must not pass `null`: by + // now other work (a failed send, an overlapping paste) may have set a + // thread error this call knows nothing about, and clearing it would + // swallow that message. + if (compressionError !== null) { + setThreadError(threadId, compressionError); + } + } finally { + const remaining = + (pendingImageCompressionsRef.current.get(threadId) ?? 0) - acceptedFiles.length; + if (remaining > 0) { + pendingImageCompressionsRef.current.set(threadId, remaining); + } else { + pendingImageCompressionsRef.current.delete(threadId); + } } - setThreadError(activeThreadId, error); }; const removeComposerImage = (imageId: string) => { @@ -2331,7 +2391,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) const imageFiles = files.filter((file) => file.type.startsWith("image/")); if (imageFiles.length === 0) return; event.preventDefault(); - addComposerImages(imageFiles); + void addComposerImages(imageFiles); }; const onComposerDragEnter = (event: React.DragEvent) => { @@ -2365,7 +2425,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) dragDepthRef.current = 0; setIsDragOverComposer(false); const files = Array.from(event.dataTransfer.files); - addComposerImages(files); + void addComposerImages(files); focusComposer(); }; diff --git a/apps/web/src/lib/stashImageCompression.test.ts b/apps/web/src/lib/imageCompression.test.ts similarity index 76% rename from apps/web/src/lib/stashImageCompression.test.ts rename to apps/web/src/lib/imageCompression.test.ts index aaffa284cbb..63712ca7e29 100644 --- a/apps/web/src/lib/stashImageCompression.test.ts +++ b/apps/web/src/lib/imageCompression.test.ts @@ -1,6 +1,11 @@ import { afterEach, describe, expect, it, vi } from "vite-plus/test"; -import { compressImageForStash, MAX_STASH_IMAGE_DATA_URL_CHARS } from "./stashImageCompression"; +import { + compressImageForStash, + compressImageToByteLimit, + MAX_COMPRESSIBLE_SOURCE_BYTES, + MAX_STASH_IMAGE_DATA_URL_CHARS, +} from "./imageCompression"; /** * jsdom has no real canvas/codec, so the re-encode path is exercised with @@ -159,6 +164,56 @@ describe("compressImageForStash", () => { }); }); + it("compressImageToByteLimit passes small files through byte-for-byte", async () => { + const bitmapSpy = vi.fn(); + vi.stubGlobal("createImageBitmap", bitmapSpy); + + const original = makeFile(1024); + const result = await compressImageToByteLimit(original, 10 * 1024 * 1024); + + expect(result.ok).toBe(true); + expect(result.ok && result.recompressed).toBe(false); + // Pass-through must be the same File object, not a copy. + expect(result.ok && result.file).toBe(original); + expect(bitmapSpy).not.toHaveBeenCalled(); + }); + + it("compressImageToByteLimit re-encodes an oversized file under the byte cap", async () => { + stubCanvasPipeline(() => 200_000); + + const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000); + + expect(result.ok).toBe(true); + expect(result.ok && result.recompressed).toBe(true); + expect(result.ok && result.file.type).toBe("image/webp"); + // The re-encoded name must match the new container format. + expect(result.ok && result.file.name).toBe("shot.webp"); + expect(result.ok && result.file.size).toBeLessThanOrEqual(1_000_000); + }); + + it("compressImageToByteLimit refuses sources above the decode-safety ceiling", async () => { + const bitmapSpy = vi.fn(); + vi.stubGlobal("createImageBitmap", bitmapSpy); + + const result = await compressImageToByteLimit( + makeFile(MAX_COMPRESSIBLE_SOURCE_BYTES + 1), + 10 * 1024 * 1024, + ); + + expect(result).toEqual({ ok: false, reason: "too-large" }); + // The whole point of the ceiling is to never decode such a file. + expect(bitmapSpy).not.toHaveBeenCalled(); + }); + + it("compressImageToByteLimit reports too-large when no encoding fits", async () => { + const { close } = stubCanvasPipeline(() => 3_000_000); + + const result = await compressImageToByteLimit(makeFile(2_000_000), 1_000_000); + + expect(result).toEqual({ ok: false, reason: "too-large" }); + expect(close).toHaveBeenCalled(); + }); + it("shrinks below the source size when the image is already under MAX_DIMENSION", async () => { // A small-but-heavy source (e.g. a dense PNG): only a real downscale can // get it under budget, since quality alone is stubbed to never suffice. diff --git a/apps/web/src/lib/stashImageCompression.ts b/apps/web/src/lib/imageCompression.ts similarity index 63% rename from apps/web/src/lib/stashImageCompression.ts rename to apps/web/src/lib/imageCompression.ts index 7f133fb2926..1c57fdad1a5 100644 --- a/apps/web/src/lib/stashImageCompression.ts +++ b/apps/web/src/lib/imageCompression.ts @@ -1,13 +1,14 @@ /** - * Re-encoding for stashed image attachments. + * Downscale + re-encode for image attachments that are too big for where + * they're headed. Two consumers share the same pipeline: * - * The composer accepts images up to `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` - * (10MB), but the stash persists them as base64 in localStorage, where the - * whole origin shares a ~5MB quota. Rather than refuse large screenshots, - * downscale + re-encode them to JPEG so a stashed prompt keeps its images. + * - The prompt stash persists images as base64 in localStorage (~5MB origin + * quota), so `compressImageForStash` targets a per-image character budget. + * - The composer accepts pasted/dropped images larger than the provider's + * `PROVIDER_SEND_TURN_MAX_IMAGE_BYTES` wire cap and shrinks them to fit + * via `compressImageToByteLimit` instead of rejecting the paste. * - * Only the *stashed copy* is compressed; the live composer attachment is - * untouched, so sending without stashing still uploads the original file. + * Images already within budget pass through untouched. */ /** @@ -17,6 +18,12 @@ const MAX_DIMENSION = 2048; /** Base64 budget for a single stashed image (~975KB of binary). */ export const MAX_STASH_IMAGE_DATA_URL_CHARS = 1_300_000; +/** + * Ceiling on the *source* file handed to the re-encoder. File size is a + * proxy for pixel count, and decoding hundreds of megapixels into an + * ImageBitmap can OOM the tab — beyond this we refuse rather than risk it. + */ +export const MAX_COMPRESSIBLE_SOURCE_BYTES = 50 * 1024 * 1024; /** * Quality ladder tried in order until the encoded image fits the budget. * The floor stays high enough to avoid visible blocking on UI screenshots; @@ -35,14 +42,18 @@ export interface CompressedStashImage { } /** - * Why an image could not be stashed. Callers report these differently: + * Why an image could not be compressed. Callers report these differently: * "too large" is a budget outcome, "unreadable" is a decode failure. */ -export type StashImageFailureReason = "too-large" | "unreadable"; +export type ImageCompressionFailureReason = "too-large" | "unreadable"; export type CompressStashImageResult = | { ok: true; image: CompressedStashImage } - | { ok: false; reason: StashImageFailureReason }; + | { ok: false; reason: ImageCompressionFailureReason }; + +export type CompressImageFileResult = + | { ok: true; file: File; recompressed: boolean } + | { ok: false; reason: ImageCompressionFailureReason }; /** Chunked so a large image can't blow the argument limit of `fromCharCode`. */ const BASE64_CHUNK_SIZE = 0x8000; @@ -73,6 +84,28 @@ function dataUrlByteLength(dataUrl: string): number { return Math.max(0, Math.floor((payload.length * 3) / 4) - padding); } +/** Base64 payload of a data URL decoded back into a `File`. */ +function dataUrlToFile(dataUrl: string, name: string, mimeType: string): File { + const payload = dataUrl.slice(dataUrl.indexOf(",") + 1); + const binary = atob(payload); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index += 1) { + bytes[index] = binary.charCodeAt(index); + } + return new File([bytes], name, { type: mimeType }); +} + +/** + * Re-encoding changes the container, so a name like `shot.png` would lie + * about its contents. Swap the extension to match the encoded mime type. + */ +function fileNameForMimeType(name: string, mimeType: string): string { + const extension = mimeType === "image/webp" ? ".webp" : ".jpg"; + const dotIndex = name.lastIndexOf("."); + const base = dotIndex > 0 ? name.slice(0, dotIndex) : name; + return `${base}${extension}`; +} + function canRecompress(): boolean { return ( typeof createImageBitmap === "function" && @@ -124,10 +157,10 @@ async function encodeToDataUrl( } /** - * Draws `bitmap` scaled to fit `maxDimension` and encodes it as JPEG, - * stepping quality down until the data URL fits `budgetChars`. Returns the - * smallest encoding produced, even if it still exceeds the budget, so the - * caller can decide whether to keep or drop it. + * Draws `bitmap` scaled to fit `maxDimension` and encodes it, stepping + * quality down until the data URL fits `budgetChars`. Returns the smallest + * encoding produced, even if it still exceeds the budget, so the caller can + * decide whether to keep or drop it. */ async function encodeWithinBudget( bitmap: ImageBitmap, @@ -165,35 +198,15 @@ async function encodeWithinBudget( return smallest; } +type ReencodeResult = + | { ok: true; dataUrl: string; mimeType: string } + | { ok: false; reason: ImageCompressionFailureReason }; + /** - * Produces the payload to persist for a stashed image. - * - * Small images are stored verbatim (preserving PNG transparency and exact - * pixels). Anything over budget is downscaled and JPEG-encoded; if it still - * doesn't fit after the fallback passes, returns `null` so the caller can - * record it as dropped. + * Shared re-encode loop: decodes `file`, then walks the quality ladder and + * fallback downscale passes until an encoding fits `budgetChars`. */ -export async function compressImageForStash( - file: File, - budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS, -): Promise { - let originalDataUrl: string; - try { - originalDataUrl = await blobToDataUrl(file); - } catch { - return { ok: false, reason: "unreadable" }; - } - if (originalDataUrl.length <= budgetChars) { - return { - ok: true, - image: { - dataUrl: originalDataUrl, - mimeType: file.type, - sizeBytes: file.size, - recompressed: false, - }, - }; - } +async function reencodeWithinBudget(file: File, budgetChars: number): Promise { if (!canRecompress()) { return { ok: false, reason: "too-large" }; } @@ -225,22 +238,14 @@ export async function compressImageForStash( // precisely *because* the target is too big (OOM on a large bitmap). // Keep trying the smaller fallback scales rather than giving up: a // reduced pass may well succeed. The exception must never escape, - // though, since the caller finalizes the entry after this returns and - // a throw would strand it as permanently "still saving". + // though, since callers finalize state after this returns and a + // throw would strand it (e.g. a stash entry stuck "still saving"). encodeFailed = true; continue; } encodeFailed = false; if (encoded && encoded.dataUrl.length <= budgetChars) { - return { - ok: true, - image: { - dataUrl: encoded.dataUrl, - mimeType: encoded.mimeType, - sizeBytes: dataUrlByteLength(encoded.dataUrl), - recompressed: true, - }, - }; + return { ok: true, dataUrl: encoded.dataUrl, mimeType: encoded.mimeType }; } } return { ok: false, reason: encodeFailed ? "unreadable" : "too-large" }; @@ -248,3 +253,83 @@ export async function compressImageForStash( bitmap.close(); } } + +/** + * Produces the payload to persist for a stashed image. + * + * Small images are stored verbatim (preserving PNG transparency and exact + * pixels). Anything over budget is downscaled and re-encoded; if it still + * doesn't fit after the fallback passes, reports a failure so the caller + * can record it as dropped. + */ +export async function compressImageForStash( + file: File, + budgetChars: number = MAX_STASH_IMAGE_DATA_URL_CHARS, +): Promise { + let originalDataUrl: string; + try { + originalDataUrl = await blobToDataUrl(file); + } catch { + return { ok: false, reason: "unreadable" }; + } + if (originalDataUrl.length <= budgetChars) { + return { + ok: true, + image: { + dataUrl: originalDataUrl, + mimeType: file.type, + sizeBytes: file.size, + recompressed: false, + }, + }; + } + const reencoded = await reencodeWithinBudget(file, budgetChars); + if (!reencoded.ok) { + return reencoded; + } + return { + ok: true, + image: { + dataUrl: reencoded.dataUrl, + mimeType: reencoded.mimeType, + sizeBytes: dataUrlByteLength(reencoded.dataUrl), + recompressed: true, + }, + }; +} + +/** + * Shrinks `file` until its binary size fits `maxBytes`, returning a new + * `File` (WebP or JPEG). Files already within the limit pass through + * untouched, preserving their exact bytes and format. Sources above + * `MAX_COMPRESSIBLE_SOURCE_BYTES` are refused outright — decoding them is + * the risk, so no amount of output budget makes them safe. + */ +export async function compressImageToByteLimit( + file: File, + maxBytes: number, +): Promise { + if (file.size <= maxBytes) { + return { ok: true, file, recompressed: false }; + } + if (file.size > MAX_COMPRESSIBLE_SOURCE_BYTES) { + return { ok: false, reason: "too-large" }; + } + // The re-encode loop budgets in data-URL characters. Base64 turns 3 bytes + // into 4 chars; flooring keeps the budget a hair conservative instead of + // admitting an encoding right at the byte cap. + const budgetChars = Math.floor(maxBytes / 3) * 4; + const reencoded = await reencodeWithinBudget(file, budgetChars); + if (!reencoded.ok) { + return reencoded; + } + return { + ok: true, + file: dataUrlToFile( + reencoded.dataUrl, + fileNameForMimeType(file.name || "image", reencoded.mimeType), + reencoded.mimeType, + ), + recompressed: true, + }; +}