diff --git a/containers/image-converter/server.mjs b/containers/image-converter/server.mjs index 2597db6e..59ccf9f5 100644 --- a/containers/image-converter/server.mjs +++ b/containers/image-converter/server.mjs @@ -8,7 +8,7 @@ import { promisify } from "node:util"; const execFileAsync = promisify(execFile); const port = 8080; -const maxInputBytes = 200 * 1024 * 1024; +const maxInputBytes = 100 * 1024 * 1024; const jpegQuality = 92; const maxOutputBytes = 1024 * 1024; const outputProfiles = [ diff --git a/containers/liteparse/server.mjs b/containers/liteparse/server.mjs index 081b50ef..d8de117f 100644 --- a/containers/liteparse/server.mjs +++ b/containers/liteparse/server.mjs @@ -19,7 +19,7 @@ const parser = new LiteParse({ quiet: true, }); const parseTimeoutMs = 90_000; -const maxInputBytes = 200 * 1024 * 1024; +const maxInputBytes = 100 * 1024 * 1024; const execFileAsync = promisify(execFile); createServer(async (request, response) => { diff --git a/docs/concepts/items.mdx b/docs/concepts/items.mdx index 847812c7..8beed6c0 100644 --- a/docs/concepts/items.mdx +++ b/docs/concepts/items.mdx @@ -36,7 +36,7 @@ ThinkEx accepts these upload families today: | Images | `.png`, `.jpg`, `.jpeg`, `.webp`, `.heic`, `.heif` | | Text documents | CSV, TSV, Markdown, code, and plain text imported as documents | -Upload limits are currently 50 files or 200 MB per selection. +Upload limits are currently 50 files or 100 MB per selection. ## Extraction and Previews diff --git a/docs/guides/import-files.mdx b/docs/guides/import-files.mdx index 1b0f6109..31b117ff 100644 --- a/docs/guides/import-files.mdx +++ b/docs/guides/import-files.mdx @@ -38,8 +38,8 @@ Use file import when you want source material to become part of a workspace. | Limit | Value | | --- | --- | | Files per selection | 50 | -| Bytes per selection | 200 MB | -| Upload concurrency | 5 | +| Bytes per selection | 100 MB | +| Upload concurrency | 3 | For AI-heavy work, upload only the sources you need for the current workspace. Smaller, well-named workspaces are easier to navigate and easier to ask about. diff --git a/src/features/workspaces/components/WorkspaceFileUploadProvider.tsx b/src/features/workspaces/components/WorkspaceFileUploadProvider.tsx index e5143b9c..26ecf482 100644 --- a/src/features/workspaces/components/WorkspaceFileUploadProvider.tsx +++ b/src/features/workspaces/components/WorkspaceFileUploadProvider.tsx @@ -43,7 +43,7 @@ export function WorkspaceFileUploadProvider({ onSuccess: (command) => { applyWorkspaceEventToCache(queryClient, command.event); }, - }).catch(() => undefined); + }); }; const requestFileSelection = (onSelectFiles: (files: File[]) => void) => { diff --git a/src/features/workspaces/files/workspace-file-upload.test.ts b/src/features/workspaces/files/workspace-file-upload.test.ts new file mode 100644 index 00000000..86a9e8b5 --- /dev/null +++ b/src/features/workspaces/files/workspace-file-upload.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { captureException, toastError, uploadFileDirectlyToR2 } = vi.hoisted(() => ({ + captureException: vi.fn(), + toastError: vi.fn(), + uploadFileDirectlyToR2: vi.fn(), +})); + +vi.mock("sonner", () => ({ + toast: { + error: toastError, + loading: vi.fn(() => "upload-toast"), + success: vi.fn(), + }, +})); + +vi.mock("#/features/workspaces/upload/workspace-file-direct-upload-client", () => ({ + uploadFileDirectlyToR2, +})); + +vi.mock("#/features/workspaces/use-workspace-client-mutation-echo", () => ({ + prepareWorkspaceClientMutationInput: (input: T) => ({ + ...input, + clientMutationId: "test-mutation", + }), +})); + +vi.mock("#/integrations/posthog/provider", () => ({ + capturePostHogClientException: captureException, +})); + +import { runWorkspaceFileUploadBatch } from "#/features/workspaces/files/workspace-file-upload"; + +beforeEach(() => { + vi.clearAllMocks(); + vi.stubGlobal( + "fetch", + vi.fn().mockImplementation(() => + Promise.resolve( + new Response( + JSON.stringify({ + completionToken: "completion-token", + uploadUrl: "https://r2.example/upload", + }), + { headers: { "content-type": "application/json" }, status: 200 }, + ), + ), + ), + ); +}); + +describe("workspace file upload batch failures", () => { + it("preserves and captures the original upload error once", async () => { + const error = new Error("Direct file upload failed because of a network error."); + uploadFileDirectlyToR2.mockRejectedValue(error); + + await runWorkspaceFileUploadBatch({ + files: [new File([new Uint8Array([1])], "paper.pdf", { type: "application/pdf" })], + onSuccess: vi.fn(), + parentId: null, + workspaceId: "workspace-id", + }); + + expect(captureException).toHaveBeenCalledOnce(); + expect(captureException).toHaveBeenCalledWith(error, { + operation: "workspace_file_upload", + upload_error_count: 1, + upload_skipped_count: 0, + upload_success_count: 0, + }); + }); + + it("does not capture a canceled upload", async () => { + const error = new DOMException("Upload canceled.", "AbortError"); + uploadFileDirectlyToR2.mockRejectedValue(error); + + await runWorkspaceFileUploadBatch({ + files: [new File([new Uint8Array([1])], "paper.pdf", { type: "application/pdf" })], + onSuccess: vi.fn(), + parentId: null, + workspaceId: "workspace-id", + }); + + expect(captureException).not.toHaveBeenCalled(); + expect(toastError).toHaveBeenCalledWith( + "Upload canceled.", + expect.objectContaining({ id: "upload-toast" }), + ); + }); + + it("does not reclassify a cache callback failure as an upload failure", async () => { + const error = new Error("Cache update failed."); + uploadFileDirectlyToR2.mockResolvedValue(undefined); + + await expect( + runWorkspaceFileUploadBatch({ + files: [new File([new Uint8Array([1])], "paper.pdf", { type: "application/pdf" })], + onSuccess: () => { + throw error; + }, + parentId: null, + workspaceId: "workspace-id", + }), + ).rejects.toBe(error); + + expect(captureException).not.toHaveBeenCalled(); + }); +}); diff --git a/src/features/workspaces/files/workspace-file-upload.ts b/src/features/workspaces/files/workspace-file-upload.ts index 791a186a..39aabfcf 100644 --- a/src/features/workspaces/files/workspace-file-upload.ts +++ b/src/features/workspaces/files/workspace-file-upload.ts @@ -1,19 +1,16 @@ -import { AsyncQueuer } from "@tanstack/pacer"; import { toast } from "sonner"; import type { WorkspaceItemSummary } from "#/features/workspaces/contracts"; import { workspaceFileUploadLimits } from "#/features/workspaces/model/workspace-file"; import type { WorkspaceCommandResult } from "#/features/workspaces/realtime/messages"; -import { - getWorkspaceUploadAbortReason, - uploadFileDirectlyToR2, -} from "#/features/workspaces/upload/workspace-file-direct-upload-client"; +import { uploadFileDirectlyToR2 } from "#/features/workspaces/upload/workspace-file-direct-upload-client"; import { partitionWorkspaceUploadSelection } from "#/features/workspaces/upload/workspace-upload-intake"; import { type CompleteWorkspaceDirectUploadInput, type WorkspaceDirectUploadSession, } from "#/features/workspaces/upload/workspace-file-upload-protocol"; import { prepareWorkspaceClientMutationInput } from "#/features/workspaces/use-workspace-client-mutation-echo"; +import { capturePostHogClientException } from "#/integrations/posthog/provider"; import { apiErrorSchema } from "#/lib/api/contracts"; import { getErrorMessage } from "#/lib/error-message"; @@ -33,17 +30,21 @@ interface WorkspaceFileUploadBatchInput { onSuccess: (command: WorkspaceCommandResult) => void; } -interface WorkspaceFileUploadBatchResult { - successCount: number; - errorCount: number; - skippedCount: number; -} +type WorkspaceFileUploadOutcome = + | { + command: WorkspaceCommandResult; + ok: true; + } + | { + error: Error; + ok: false; + }; const uploadRequestTimeoutMs = 5 * 60_000; export async function runWorkspaceFileUploadBatch( input: WorkspaceFileUploadBatchInput, -): Promise { +): Promise { const { accepted, rejected } = partitionWorkspaceUploadSelection(input.files); for (const rejection of rejected) { @@ -51,23 +52,28 @@ export async function runWorkspaceFileUploadBatch( } if (accepted.length === 0) { - return { - successCount: 0, - errorCount: 0, - skippedCount: rejected.length, - }; + return; } const controller = new AbortController(); + const cancelAction = { + label: "Cancel", + onClick: () => controller.abort(new DOMException("Upload canceled.", "AbortError")), + }; const totalBytes = accepted.reduce((total, file) => total + file.size, 0); const loadedBytesByFile = new Map(accepted.map((file) => [file, 0])); const toastId = toast.loading(getUploadBatchStageMessage("uploading", accepted, 0), { - action: { - label: "Cancel", - onClick: () => controller.abort(new DOMException("Upload canceled.", "AbortError")), - }, + action: cancelAction, duration: Number.POSITIVE_INFINITY, }); + const showUploadError = (error: unknown) => { + toast.error(getUploadBatchErrorMessage(error, controller.signal), { + action: undefined, + description: undefined, + duration: 5_000, + id: toastId, + }); + }; let lastProgressPercent = -1; const onProgress = (file: File, loadedBytes: number) => { loadedBytesByFile.set(file, Math.min(file.size, loadedBytes)); @@ -84,10 +90,7 @@ export async function runWorkspaceFileUploadBatch( toast.loading( getUploadBatchStageMessage(percent === 100 ? "finalizing" : "uploading", accepted, percent), { - action: { - label: "Cancel", - onClick: () => controller.abort(new DOMException("Upload canceled.", "AbortError")), - }, + action: cancelAction, duration: Number.POSITIVE_INFINITY, id: toastId, }, @@ -95,7 +98,7 @@ export async function runWorkspaceFileUploadBatch( }; try { - const result = await uploadAcceptedFiles({ + const outcomes = await uploadAcceptedFiles({ files: accepted, onProgress, onSuccess: input.onSuccess, @@ -103,42 +106,39 @@ export async function runWorkspaceFileUploadBatch( signal: controller.signal, workspaceId: input.workspaceId, }); + const failures = outcomes.flatMap((outcome) => (outcome.ok ? [] : [outcome.error])); + const successCount = outcomes.length - failures.length; + const reportableFailure = failures.find((failure) => !isWorkspaceUploadAbortError(failure)); + + if (reportableFailure) { + capturePostHogClientException(reportableFailure, { + operation: "workspace_file_upload", + upload_error_count: failures.length, + upload_skipped_count: rejected.length, + upload_success_count: successCount, + }); + } - toast.success(getUploadBatchSuccessMessage(result, accepted.length), { + if (successCount === 0) { + showUploadError(failures[0]); + return; + } + + toast.success(getUploadBatchSuccessMessage(successCount, failures.length, accepted.length), { action: undefined, description: undefined, duration: 3_000, id: toastId, }); - - return { - ...result, - skippedCount: rejected.length, - }; } catch (error) { - toast.error(getUploadBatchErrorMessage(error, controller.signal), { - action: undefined, - description: undefined, - duration: 5_000, - id: toastId, - }); + showUploadError(error); throw error; } } -async function postWorkspaceFileUpload( +async function uploadWorkspaceFile( job: WorkspaceFileUploadJob, ): Promise> { - const uploadResponse = await postWorkspaceDirectUpload(job); - - if (!uploadResponse.ok) { - throw new Error(await getWorkspaceFileUploadErrorMessage(uploadResponse)); - } - - return (await uploadResponse.json()) as WorkspaceCommandResult; -} - -async function postWorkspaceDirectUpload(job: WorkspaceFileUploadJob) { const endpoint = `/api/v1/workspaces/${job.workspaceId}/file-upload`; const contentType = job.file.type || "application/octet-stream"; const session = await requestUploadJson( @@ -165,15 +165,30 @@ async function postWorkspaceDirectUpload(job: WorkspaceFileUploadJob) { url: session.uploadUrl, }); - const completeInput: CompleteWorkspaceDirectUploadInput = { - completionToken: session.completionToken, - }; - return fetch(`${endpoint}?action=complete`, { - body: JSON.stringify(completeInput), - headers: { "content-type": "application/json" }, - method: "POST", - signal: getUploadRequestSignal(job.signal), - }); + return requestUploadJson>( + `${endpoint}?action=complete`, + { + body: JSON.stringify({ + completionToken: session.completionToken, + } satisfies CompleteWorkspaceDirectUploadInput), + headers: { "content-type": "application/json" }, + method: "POST", + signal: getUploadRequestSignal(job.signal), + }, + ); +} + +async function settleWorkspaceFileUpload( + job: WorkspaceFileUploadJob, +): Promise { + try { + return { command: await uploadWorkspaceFile(job), ok: true }; + } catch (error) { + return { + error: error instanceof Error ? error : new Error("Unable to upload file."), + ok: false, + }; + } } async function requestUploadJson(url: string, init: RequestInit): Promise { @@ -186,74 +201,47 @@ async function requestUploadJson(url: string, init: RequestInit): Promise return (await response.json()) as T; } -function toUploadJob(input: { - workspaceId: string; - parentId: string | null; - file: File; - clientMutationId?: string; - onProgress: (loadedBytes: number) => void; - signal: AbortSignal; -}): WorkspaceFileUploadJob { - return { - ...prepareWorkspaceClientMutationInput(input), - onProgress: input.onProgress, - signal: input.signal, - }; -} - -function uploadAcceptedFiles(input: { +async function uploadAcceptedFiles(input: { workspaceId: string; parentId: string | null; files: readonly File[]; onProgress: (file: File, loadedBytes: number) => void; onSuccess: (command: WorkspaceCommandResult) => void; signal: AbortSignal; -}): Promise> { +}): Promise { const jobs = input.files.map((file) => - toUploadJob({ + prepareWorkspaceClientMutationInput({ file, - onProgress: (loadedBytes) => input.onProgress(file, loadedBytes), + onProgress: (loadedBytes: number) => input.onProgress(file, loadedBytes), parentId: input.parentId, signal: input.signal, workspaceId: input.workspaceId, }), ); - const total = jobs.length; - - return new Promise((resolve, reject) => { - new AsyncQueuer(postWorkspaceFileUpload, { - concurrency: workspaceFileUploadLimits.concurrency, - throwOnError: false, - initialItems: jobs, - onSuccess: (command) => { - input.onSuccess(command); - }, - onSettled: (_item, queuer) => { - if (queuer.store.state.settledCount < total) { - return; - } - - const { successCount, errorCount } = queuer.store.state; - - if (successCount === 0) { - if (input.signal.aborted) { - reject(getWorkspaceUploadAbortReason(input.signal)); - return; - } - reject( - new Error( - total === 1 - ? `Failed to upload ${input.files[0]?.name ?? "file"}.` - : `Failed to upload ${total} files.`, - ), - ); - return; - } - - resolve({ successCount, errorCount }); - }, - }); - }); + let nextJobIndex = 0; + + const runWorker = async () => { + const outcomes: WorkspaceFileUploadOutcome[] = []; + + while (true) { + const job = jobs[nextJobIndex++]; + + if (!job) { + return outcomes; + } + + const outcome = await settleWorkspaceFileUpload(job); + + if (outcome.ok) { + input.onSuccess(outcome.command); + } + outcomes.push(outcome); + } + }; + const workerCount = Math.min(workspaceFileUploadLimits.concurrency, jobs.length); + const workerOutcomes = await Promise.all(Array.from({ length: workerCount }, () => runWorker())); + + return workerOutcomes.flat(); } async function getWorkspaceFileUploadErrorMessage(response: Response) { @@ -292,21 +280,22 @@ function getUploadBatchErrorMessage(error: unknown, signal: AbortSignal) { return getErrorMessage(error, "Unable to upload files right now."); } +function isWorkspaceUploadAbortError(error: Error) { + return error instanceof DOMException && error.name === "AbortError"; +} + function getUploadRequestSignal(signal: AbortSignal) { return AbortSignal.any([signal, AbortSignal.timeout(uploadRequestTimeoutMs)]); } -function getUploadBatchSuccessMessage( - result: Pick, - total: number, -) { +function getUploadBatchSuccessMessage(successCount: number, errorCount: number, total: number) { if (total === 1) { return "Uploaded 1 file."; } - if (result.errorCount === 0) { - return `Uploaded ${result.successCount} files.`; + if (errorCount === 0) { + return `Uploaded ${successCount} files.`; } - return `Uploaded ${result.successCount} of ${total} files.`; + return `Uploaded ${successCount} of ${total} files.`; } diff --git a/src/features/workspaces/model/workspace-file/limits.ts b/src/features/workspaces/model/workspace-file/limits.ts index 1d808868..4fbc6811 100644 --- a/src/features/workspaces/model/workspace-file/limits.ts +++ b/src/features/workspaces/model/workspace-file/limits.ts @@ -1,7 +1,7 @@ export const workspaceFileUploadLimits = { maxFilesPerSelection: 50, - maxFileBytes: 200 * 1024 * 1024, - maxSelectionBytes: 200 * 1024 * 1024, + maxFileBytes: 100 * 1024 * 1024, + maxSelectionBytes: 100 * 1024 * 1024, maxDocumentImportBytes: 10 * 1024 * 1024, concurrency: 3, } as const; diff --git a/src/features/workspaces/upload/workspace-upload-intake.test.ts b/src/features/workspaces/upload/workspace-upload-intake.test.ts index bead0a2a..ae6bedb6 100644 --- a/src/features/workspaces/upload/workspace-upload-intake.test.ts +++ b/src/features/workspaces/upload/workspace-upload-intake.test.ts @@ -9,7 +9,7 @@ import { } from "#/features/workspaces/upload/workspace-upload-intake"; describe("workspace upload intake", () => { - it("accepts a binary file at the 200 MB limit", () => { + it("accepts a binary file at the 100 MB limit", () => { expect( validateWorkspaceUpload({ contentType: "application/pdf", @@ -19,6 +19,19 @@ describe("workspace upload intake", () => { ).toMatchObject({ ok: true, plan: { kind: "file" } }); }); + it("rejects a binary file above the 100 MB limit", () => { + expect( + validateWorkspaceUpload({ + contentType: "application/pdf", + fileName: "research.pdf", + sizeBytes: workspaceFileUploadLimits.maxFileBytes + 1, + }), + ).toMatchObject({ + error: { code: "SELECTION_TOO_LARGE", status: 413 }, + ok: false, + }); + }); + it("bounds document imports before they are materialized in Worker memory", () => { expect( validateWorkspaceUpload({ diff --git a/src/features/workspaces/upload/workspace-upload-intake.ts b/src/features/workspaces/upload/workspace-upload-intake.ts index 2636b7b9..4fa52459 100644 --- a/src/features/workspaces/upload/workspace-upload-intake.ts +++ b/src/features/workspaces/upload/workspace-upload-intake.ts @@ -145,7 +145,7 @@ export function validateWorkspaceUpload(input: { return { error: { code: "SELECTION_TOO_LARGE", - message: "Upload up to 200 MB at once.", + message: "Upload up to 100 MB at once.", status: 413, }, ok: false, @@ -199,7 +199,7 @@ export function getWorkspaceUploadSelectionValidationError(input: { if (input.selectionBytes + input.file.size > workspaceFileUploadLimits.maxSelectionBytes) { return { code: "SELECTION_TOO_LARGE", - message: "Upload up to 200 MB at once.", + message: "Upload up to 100 MB at once.", status: 413, }; } diff --git a/src/features/workspaces/use-workspace-client-mutation-echo.ts b/src/features/workspaces/use-workspace-client-mutation-echo.ts index 40074a77..799da720 100644 --- a/src/features/workspaces/use-workspace-client-mutation-echo.ts +++ b/src/features/workspaces/use-workspace-client-mutation-echo.ts @@ -48,9 +48,9 @@ export function shouldIgnoreWorkspaceClientMutationEcho(event: WorkspaceRealtime return true; } -export function prepareWorkspaceClientMutationInput( - input: TInput, -) { +export function prepareWorkspaceClientMutationInput( + input: TInput & WorkspaceClientMutationInput, +): TInput & { clientMutationId: string } { const clientMutationId = input.clientMutationId ?? crypto.randomUUID(); trackWorkspaceClientMutationId(clientMutationId); diff --git a/src/integrations/posthog/provider.tsx b/src/integrations/posthog/provider.tsx index cd9d8d8e..a9b266e1 100644 --- a/src/integrations/posthog/provider.tsx +++ b/src/integrations/posthog/provider.tsx @@ -77,20 +77,6 @@ export function capturePostHogClientException(error: Error, properties?: Record< posthog.captureException(error, properties); } -function normalizeBrowserError(value: unknown, fallbackMessage: string) { - if (value instanceof Error) { - return value; - } - - if (typeof value === "string" && value.trim()) { - return new Error(value); - } - - const error = new Error(fallbackMessage); - (error as Error & { cause?: unknown }).cause = value; - return error; -} - type AuthenticatedSession = NonNullable; function identifyPostHogUser(session: AuthenticatedSession) { @@ -137,43 +123,6 @@ function PostHogAuthSync() { return null; } -function PostHogGlobalErrorCapture() { - useEffect(() => { - const handleError = (event: ErrorEvent) => { - capturePostHogClientException( - normalizeBrowserError(event.error ?? event.message, "Unhandled browser error"), - { - colno: event.colno, - error_boundary: "window.error", - filename: event.filename, - lineno: event.lineno, - message: event.message, - }, - ); - }; - - const handleUnhandledRejection = (event: PromiseRejectionEvent) => { - capturePostHogClientException( - normalizeBrowserError(event.reason, "Unhandled promise rejection"), - { - error_boundary: "window.unhandledrejection", - reason_type: typeof event.reason, - }, - ); - }; - - window.addEventListener("error", handleError); - window.addEventListener("unhandledrejection", handleUnhandledRejection); - - return () => { - window.removeEventListener("error", handleError); - window.removeEventListener("unhandledrejection", handleUnhandledRejection); - }; - }, []); - - return null; -} - export default function PostHogProvider({ children }: { children: ReactNode }) { if (!isPostHogEnabled) { return children; @@ -181,7 +130,6 @@ export default function PostHogProvider({ children }: { children: ReactNode }) { return ( - {children}