From e2e94abf9f41f7f70b988910646649e1eeda4afb Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:47:34 +1000 Subject: [PATCH 1/5] feat(ai): native Files API support across providers (upload adapters + `file` content source) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add first-class support for provider Files / storage APIs so callers can upload media once and reference it by a provider-issued handle instead of re-sending base64 or a public URL each request. - New tree-shakeable `files` adapter kind: openaiFiles(), anthropicFiles(), geminiFiles(), falFiles() — each with upload(), plus get()/delete() where the provider has a lifecycle API (fal is upload-only). Driven by the new uploadFile()/getFile()/deleteFile() activity functions. - New `{ type: 'file' }` arm on ContentPartSource. Adapters map it to the provider's native reference: OpenAI (Responses) input_image/input_file file_id, Anthropic file_id source (sends the files-api-2025-04-14 beta), Gemini fileData.fileUri, fal storage URL. fileSourceFromHandle() builds the source from an uploaded FileHandle. - Runtime provider routing: a handle only routes to its issuing provider; cross-provider handles and endpoints that require raw bytes (image edits, Veo, Chat Completions images, Bedrock, Mistral, Grok, OpenRouter, Ollama) throw a clear error instead of silently mis-mapping. Closes #909 Co-Authored-By: Claude Opus 4.8 (1M context) --- .changeset/native-files-api-support.md | 22 +++ docs/advanced/files-api.md | 124 ++++++++++++++++ docs/advanced/multimodal-content.md | 30 ++++ docs/config.json | 8 +- packages/ai-anthropic/src/adapters/files.ts | 82 +++++++++++ packages/ai-anthropic/src/adapters/text.ts | 131 +++++++++++------ packages/ai-anthropic/src/index.ts | 8 ++ .../src/text/text-provider-options.ts | 8 +- .../ai-anthropic/tests/files-source.test.ts | 109 ++++++++++++++ .../src/converse/message-converter.ts | 8 +- packages/ai-event-client/src/index.ts | 12 +- packages/ai-fal/src/adapters/files.ts | 57 ++++++++ packages/ai-fal/src/adapters/video.ts | 14 +- packages/ai-fal/src/image/image-inputs.ts | 28 +++- packages/ai-fal/src/index.ts | 6 + .../tests/content-source-to-fal-url.test.ts | 37 +++++ packages/ai-gemini/src/adapters/files.ts | 84 +++++++++++ packages/ai-gemini/src/adapters/image.ts | 11 +- packages/ai-gemini/src/adapters/text.ts | 16 ++- packages/ai-gemini/src/adapters/video.ts | 14 +- .../experimental/text-interactions/adapter.ts | 7 +- packages/ai-gemini/src/index.ts | 8 ++ packages/ai-grok/src/adapters/image.ts | 7 +- packages/ai-grok/src/adapters/video.ts | 7 +- packages/ai-mistral/src/adapters/text.ts | 4 + packages/ai-ollama/src/adapters/text.ts | 14 +- packages/ai-openai/src/adapters/files.ts | 86 +++++++++++ .../src/image/image-input-to-file.ts | 10 ++ packages/ai-openai/src/index.ts | 8 ++ packages/ai-openai/tests/files-source.test.ts | 112 +++++++++++++++ packages/ai-openrouter/src/adapters/image.ts | 7 +- .../src/adapters/responses-text.ts | 10 +- packages/ai-openrouter/src/adapters/text.ts | 10 +- packages/ai/src/activities/files/adapter.ts | 106 ++++++++++++++ packages/ai/src/activities/files/index.ts | 94 +++++++++++++ packages/ai/src/activities/index.ts | 26 +++- packages/ai/src/client.ts | 1 + packages/ai/src/index.ts | 13 ++ packages/ai/src/types.ts | 43 +++++- packages/ai/src/utilities/content-source.ts | 57 ++++++++ packages/ai/src/utilities/tool-result.ts | 4 +- packages/ai/tests/files-source.test.ts | 133 ++++++++++++++++++ .../src/adapters/chat-completions-text.ts | 16 ++- .../src/adapters/responses-text.ts | 35 ++++- 44 files changed, 1545 insertions(+), 82 deletions(-) create mode 100644 .changeset/native-files-api-support.md create mode 100644 docs/advanced/files-api.md create mode 100644 packages/ai-anthropic/src/adapters/files.ts create mode 100644 packages/ai-anthropic/tests/files-source.test.ts create mode 100644 packages/ai-fal/src/adapters/files.ts create mode 100644 packages/ai-fal/tests/content-source-to-fal-url.test.ts create mode 100644 packages/ai-gemini/src/adapters/files.ts create mode 100644 packages/ai-openai/src/adapters/files.ts create mode 100644 packages/ai-openai/tests/files-source.test.ts create mode 100644 packages/ai/src/activities/files/adapter.ts create mode 100644 packages/ai/src/activities/files/index.ts create mode 100644 packages/ai/src/utilities/content-source.ts create mode 100644 packages/ai/tests/files-source.test.ts diff --git a/.changeset/native-files-api-support.md b/.changeset/native-files-api-support.md new file mode 100644 index 000000000..361362ba5 --- /dev/null +++ b/.changeset/native-files-api-support.md @@ -0,0 +1,22 @@ +--- +'@tanstack/ai': minor +'@tanstack/ai-event-client': minor +'@tanstack/openai-base': minor +'@tanstack/ai-openai': minor +'@tanstack/ai-anthropic': minor +'@tanstack/ai-gemini': minor +'@tanstack/ai-fal': minor +'@tanstack/ai-mistral': patch +'@tanstack/ai-grok': patch +'@tanstack/ai-openrouter': patch +'@tanstack/ai-ollama': patch +'@tanstack/ai-bedrock': patch +--- + +feat(ai): native Files API support across providers (upload adapters + `file` content source) + +Adds first-class support for provider **Files / storage APIs** so callers can upload media once and reference it by a provider-issued handle instead of re-sending base64 or a public URL each request (lower latency/bandwidth, no re-buffering on memory-constrained runtimes). + +- **New tree-shakeable `files` adapter kind** — `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, and `falFiles()`. Each exposes `upload()`, and (where the provider has a lifecycle API) `get()` / `delete()`. Drive them with the new `uploadFile()` / `getFile()` / `deleteFile()` activity functions. fal is upload-only. +- **New `{ type: 'file' }` arm on `ContentPartSource`** — reference an uploaded handle in a chat message. Adapters map it to the right wire field: OpenAI (Responses) `input_image`/`input_file` `file_id`, Anthropic `file_id` message source (with the `files-api-2025-04-14` beta), Gemini `fileData.fileUri`, fal storage URL passthrough. Use `fileSourceFromHandle(handle)` to build the source from an uploaded `FileHandle`. +- **Runtime provider routing** — a file handle only routes to the provider that issued it; adapters throw a clear error on a cross-provider handle, and providers/endpoints that can't consume a handle (image edits, Veo, Chat Completions images, Bedrock, Mistral, Grok, OpenRouter, Ollama) throw a clear "unsupported file source" error instead of silently mis-mapping. diff --git a/docs/advanced/files-api.md b/docs/advanced/files-api.md new file mode 100644 index 000000000..fcf5d260c --- /dev/null +++ b/docs/advanced/files-api.md @@ -0,0 +1,124 @@ +--- +title: Files API +id: files-api +description: "Upload media once and reference it by a provider-issued handle with TanStack AI's tree-shakeable files adapters (OpenAI, Anthropic, Gemini, fal)." +keywords: + - tanstack ai + - files api + - file upload + - file_id + - fileData + - multimodal +--- + +Provider **Files / storage APIs** let you upload a media asset once and reference it later by a lightweight handle, instead of re-sending base64 (or relying on the provider to re-fetch a public URL) on every request. That means large or reused inputs are uploaded a single time — lower latency and bandwidth, no re-buffering of base64 on memory-constrained runtimes (e.g. Cloudflare Workers) — plus access to provider-side file lifecycle (TTL, deletion). + +TanStack AI exposes this as a tree-shakeable **`files` adapter** per provider, paired with a `{ type: 'file' }` [content source](./multimodal-content.md#file-handle-files-api) you drop into a message. + +## Files adapters + +Each provider with a native surface has a factory: `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, and `falFiles()`. They read the same API-key env var as the provider's other adapters, or accept an explicit key. + +```typescript +import { openaiFiles } from '@tanstack/ai-openai' +import { geminiFiles } from '@tanstack/ai-gemini' +import { anthropicFiles } from '@tanstack/ai-anthropic' +import { falFiles } from '@tanstack/ai-fal' + +const files = openaiFiles() // reads OPENAI_API_KEY +``` + +### upload + +`upload()` accepts a `Blob` (memory-efficient — preferred for large assets) or `{ data, mimeType }` where `data` is base64. It returns a `FileHandle`: + +```typescript +const handle = await openaiFiles().upload({ + data: pdfBase64, + mimeType: 'application/pdf', +}) +// handle: { id, provider, uri?, mimeType?, sizeBytes?, expiresAt?, filename? } +``` + +- `id` — the provider handle used for `get` / `delete` (OpenAI/Anthropic `file_id`, Gemini file resource name, fal storage URL). +- `uri` — the handle's URL form when the provider exposes one (Gemini file URI, fal storage URL); `undefined` for OpenAI/Anthropic, whose handles are opaque ids. +- `expiresAt` — epoch milliseconds, when the provider schedules the handle to expire. + +### get and delete + +Providers with a lifecycle API expose `get()` and `delete()`: + +```typescript +const meta = await openaiFiles().get(handle.id) +await openaiFiles().delete(handle.id) +``` + +> fal storage is **upload-only** — `falFiles()` has no `get` / `delete`, and calling them throws a clear error. + +## Referencing a handle in a message + +Use `fileSourceFromHandle(handle)` to turn a `FileHandle` into a `{ type: 'file' }` content source. Each adapter maps it to the provider's native reference (OpenAI/Anthropic `file_id`, Gemini `fileData.fileUri`, fal storage URL). A handle only works with the provider that issued it — passing it elsewhere throws. + +### Server: upload + reference + +```typescript +import { chat, fileSourceFromHandle } from '@tanstack/ai' +import { anthropicText } from '@tanstack/ai-anthropic' +import { anthropicFiles } from '@tanstack/ai-anthropic' + +export async function askAboutPdf(pdfBase64: string, request: string) { + // Upload once; reuse the handle across turns. + const handle = await anthropicFiles().upload({ + data: pdfBase64, + mimeType: 'application/pdf', + }) + + return chat({ + adapter: anthropicText('claude-sonnet-5'), + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: request }, + { type: 'document', source: fileSourceFromHandle(handle) }, + ], + }, + ], + }) +} +``` + +### Client: reuse a handle across requests + +Upload happens server-side (it needs the provider key), so the client works with the returned handle. Persist `{ id, provider, uri, mimeType }` and rebuild the source on each turn: + +```typescript +import { fileSourceFromHandle } from '@tanstack/ai' +import type { FileHandle } from '@tanstack/ai' + +// `handle` was returned by your server's upload endpoint and stored client-side. +function imageMessage(handle: FileHandle, prompt: string) { + return { + role: 'user' as const, + content: [ + { type: 'text' as const, content: prompt }, + { type: 'image' as const, source: fileSourceFromHandle(handle) }, + ], + } +} +``` + +## Provider support + +| Provider | Adapter | Handle referenced as | Lifecycle | +| --- | --- | --- | --- | +| OpenAI | `openaiFiles()` | Responses `input_image` / `input_file` `file_id` | `get`, `delete` | +| Anthropic | `anthropicFiles()` | `file_id` message source (sends the `files-api-2025-04-14` beta) | `get`, `delete` | +| Gemini | `geminiFiles()` | `fileData.fileUri` (the handle URI) | `get`, `delete` | +| fal | `falFiles()` | storage URL (used like any URL) | upload-only | + +Gemini and fal handles are URLs, so they also round-trip through a plain `{ type: 'url' }` source; OpenAI and Anthropic handles are opaque ids that require the `{ type: 'file' }` source. + +### Endpoints that require raw bytes + +Some endpoints have no "reference an uploaded handle" option — OpenAI's `images/edits` and Sora `input_reference`, and Gemini's Veo, need the actual bytes (or, for Veo, a `gs://` URI). The OpenAI **Chat Completions** image path also references images only by URL/data URI, not `file_id` — use the Responses adapter (`openaiText`) for `file_id` images. Passing a `{ type: 'file' }` source to any of these throws a clear error rather than silently mis-mapping. diff --git a/docs/advanced/multimodal-content.md b/docs/advanced/multimodal-content.md index 0bea1a795..030cd899f 100644 --- a/docs/advanced/multimodal-content.md +++ b/docs/advanced/multimodal-content.md @@ -258,6 +258,36 @@ const imagePart = { **Note:** Not all providers support URL-based content for all modalities. Check provider documentation for specifics. +### File Handle (Files API) + +Use `type: 'file'` to reference media you uploaded once via a provider's [Files API](./files-api.md) — the provider stores the bytes and you pass a lightweight handle instead of re-sending base64 or a public URL every request. A handle only works with the provider that issued it, so the `provider` field is required and validated at request time. + +```typescript +import { openaiFiles } from '@tanstack/ai-openai' +import { openaiText, chat, fileSourceFromHandle } from '@tanstack/ai' + +// Upload once... +const handle = await openaiFiles().upload({ data: pdfBase64, mimeType: 'application/pdf' }) + +// ...then reference the handle by id in as many requests as you like. +for await (const chunk of chat({ + adapter: openaiText('gpt-5.5'), + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { type: 'document', source: fileSourceFromHandle(handle) }, + ], + }, + ], +})) { + // ... +} +``` + +`fileSourceFromHandle(handle)` builds the `{ type: 'file', value, provider }` source for you (picking the handle URL for Gemini/fal or the opaque id for OpenAI/Anthropic). Each adapter maps it to the provider's native reference (`file_id`, `fileData.fileUri`, or storage URL). Passing a handle to a different provider — or to an endpoint that requires raw bytes (image edits, Veo) — throws a clear error. See [Files API](./files-api.md) for uploading, retrieving, and deleting handles. + ## Backward Compatibility String content continues to work as before: diff --git a/docs/config.json b/docs/config.json index 687eae58e..9ca2ff725 100644 --- a/docs/config.json +++ b/docs/config.json @@ -636,7 +636,13 @@ { "label": "Multimodal Content", "to": "advanced/multimodal-content", - "addedAt": "2026-04-15" + "addedAt": "2026-04-15", + "updatedAt": "2026-07-08" + }, + { + "label": "Files API", + "to": "advanced/files-api", + "addedAt": "2026-07-08" }, { "label": "Per-Model Type Safety", diff --git a/packages/ai-anthropic/src/adapters/files.ts b/packages/ai-anthropic/src/adapters/files.ts new file mode 100644 index 000000000..b0b5f6bcd --- /dev/null +++ b/packages/ai-anthropic/src/adapters/files.ts @@ -0,0 +1,82 @@ +import { toFile } from '@anthropic-ai/sdk' +import { + BaseFilesAdapter, + normalizeFileUploadInput, +} from '@tanstack/ai/adapters' +import { createAnthropicClient, getAnthropicApiKeyFromEnv } from '../utils/client' +import type Anthropic_SDK from '@anthropic-ai/sdk' +import type { FileMetadata } from '@anthropic-ai/sdk/resources/beta/files' +import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' +import type { AnthropicClientConfig } from '../utils/client' + +/** Beta header required for the Anthropic Files API. */ +const FILES_API_BETA = 'files-api-2025-04-14' + +export interface AnthropicFilesConfig extends AnthropicClientConfig {} + +/** + * Anthropic Files adapter — uploads media to the Anthropic Files API (beta) and + * references it by `file_id`. Pair with `anthropicText()`: reference the + * returned handle in an image/document message via `fileSourceFromHandle`. + */ +export class AnthropicFilesAdapter extends BaseFilesAdapter { + readonly name = 'anthropic' as const + private readonly client: Anthropic_SDK + + constructor(config: AnthropicFilesConfig) { + super() + this.client = createAnthropicClient(config) + } + + async upload(input: FileUploadInput): Promise { + const { blob, mimeType, filename } = normalizeFileUploadInput(input) + const file = await toFile(blob, filename, { + ...(mimeType ? { type: mimeType } : {}), + }) + const result = await this.client.beta.files.upload({ + file, + betas: [FILES_API_BETA], + }) + return toFileHandle(result) + } + + async get(id: string): Promise { + const result = await this.client.beta.files.retrieveMetadata(id, { + betas: [FILES_API_BETA], + }) + return toFileHandle(result) + } + + async delete(id: string): Promise { + await this.client.beta.files.delete(id, { betas: [FILES_API_BETA] }) + } +} + +function toFileHandle(file: FileMetadata): FileHandle { + return { + id: file.id, + provider: 'anthropic', + mimeType: file.mime_type, + sizeBytes: file.size_bytes, + filename: file.filename, + } +} + +/** + * Create an Anthropic Files adapter with an explicit API key. + */ +export function createAnthropicFiles( + apiKey: string, + config?: Omit, +): AnthropicFilesAdapter { + return new AnthropicFilesAdapter({ apiKey, ...config }) +} + +/** + * Create an Anthropic Files adapter, reading the API key from `ANTHROPIC_API_KEY`. + */ +export function anthropicFiles( + config?: Omit, +): AnthropicFilesAdapter { + return createAnthropicFiles(getAnthropicApiKeyFromEnv(), config) +} diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index d431f5486..0c77e316a 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -1,4 +1,9 @@ -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + assertOwnFileSource, + isFileSource, + normalizeSystemPrompts, +} from '@tanstack/ai' import { toRunErrorRawEvent } from '@tanstack/ai/adapter-internals' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' @@ -29,20 +34,25 @@ import type { } from '@tanstack/ai/adapters' import type { InternalLogger } from '@tanstack/ai/adapter-internals' import type { - Base64ImageSource, - Base64PDFSource, - ContentBlockParam, - DocumentBlockParam, - ImageBlockParam, ServerToolUseBlockParam, TextBlockParam, ThinkingBlockParam, ToolUseBlockParam, - URLImageSource, - URLPDFSource, WebFetchToolResultBlockParam, WebSearchToolResultBlockParam, } from '@anthropic-ai/sdk/resources/messages' +import type { + BetaBase64ImageSource, + BetaBase64PDFSource, + BetaContentBlockParam, + BetaFileDocumentSource, + BetaFileImageSource, + BetaImageBlockParam, + BetaRequestDocumentBlock, + BetaTextBlockParam, + BetaURLImageSource, + BetaURLPDFSource, +} from '@anthropic-ai/sdk/resources/beta/messages' import type Anthropic_SDK from '@anthropic-ai/sdk' import type { AnthropicBeta } from '@anthropic-ai/sdk/resources/beta/beta' import type { @@ -143,11 +153,28 @@ function buildServerToolResultBlock( } } +/** + * True when any message carries a provider file-handle source, so the request + * must send the Files API beta header. + */ +export function messagesHaveFileSource( + messages: Array, +): boolean { + return messages.some( + (message) => + Array.isArray(message.content) && + message.content.some( + (part) => 'source' in part && isFileSource(part.source), + ), + ) +} + /** * Computes the `betas` array for a Messages request. Unions: * - `interleaved-thinking-2025-05-14` when interleaved thinking is enabled, * - `code-execution-2025-08-25` when a `code_execution` tool is present, - * - `skills-2025-10-02` when that tool carries skills. + * - `skills-2025-10-02` when that tool carries skills, + * - `files-api-2025-04-14` when a message references an uploaded file handle. * Returns `undefined` when none apply (so the call site omits `betas`). */ export function computeAnthropicBetas( @@ -160,9 +187,12 @@ export function computeAnthropicBetas( } } | undefined, + hasFileSource = false, ): Array | undefined { const betas = new Set() + if (hasFileSource) betas.add('files-api-2025-04-14') + const useInterleavedThinking = modelOptions?.thinking?.type === 'enabled' && typeof modelOptions.thinking.budget_tokens === 'number' && @@ -287,7 +317,11 @@ export class AnthropicTextAdapter< // `betas` is attached at the call site rather than in the shared mapper // because the beta set depends on both the tools and the modelOptions. - const betas = computeAnthropicBetas(options.tools, options.modelOptions) + const betas = computeAnthropicBetas( + options.tools, + options.modelOptions, + messagesHaveFileSource(options.messages), + ) // `client.beta.messages` is Anthropic's permanent staging surface, not a // sunset path: it's a superset of `client.messages` that additionally @@ -377,6 +411,7 @@ export class AnthropicTextAdapter< const betas = computeAnthropicBetas( chatOptions.tools, chatOptions.modelOptions, + messagesHaveFileSource(chatOptions.messages), ) // Make non-streaming request with tool_choice forced to our structured output tool const response = await this.client.beta.messages.create( @@ -617,7 +652,7 @@ export class AnthropicTextAdapter< private convertContentPartToAnthropic( part: ContentPart, - ): TextBlockParam | ImageBlockParam | DocumentBlockParam { + ): BetaTextBlockParam | BetaImageBlockParam | BetaRequestDocumentBlock { switch (part.type) { case 'text': { const metadata = part.metadata as AnthropicTextMetadata | undefined @@ -630,21 +665,29 @@ export class AnthropicTextAdapter< case 'image': { const metadata = part.metadata as AnthropicImageMetadata | undefined - const imageSource: Base64ImageSource | URLImageSource = - part.source.type === 'data' - ? { - type: 'base64', - data: part.source.value, - media_type: part.source.mimeType as - | 'image/jpeg' - | 'image/png' - | 'image/gif' - | 'image/webp', - } - : { - type: 'url', - url: part.source.value, - } + let imageSource: + | BetaBase64ImageSource + | BetaURLImageSource + | BetaFileImageSource + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + imageSource = { type: 'file', file_id: part.source.value } + } else if (part.source.type === 'data') { + imageSource = { + type: 'base64', + data: part.source.value, + media_type: part.source.mimeType as + | 'image/jpeg' + | 'image/png' + | 'image/gif' + | 'image/webp', + } + } else { + imageSource = { + type: 'url', + url: part.source.value, + } + } return { type: 'image', source: imageSource, @@ -653,17 +696,25 @@ export class AnthropicTextAdapter< } case 'document': { const metadata = part.metadata as AnthropicDocumentMetadata | undefined - const docSource: Base64PDFSource | URLPDFSource = - part.source.type === 'data' - ? { - type: 'base64', - data: part.source.value, - media_type: part.source.mimeType as 'application/pdf', - } - : { - type: 'url', - url: part.source.value, - } + let docSource: + | BetaBase64PDFSource + | BetaURLPDFSource + | BetaFileDocumentSource + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + docSource = { type: 'file', file_id: part.source.value } + } else if (part.source.type === 'data') { + docSource = { + type: 'base64', + data: part.source.value, + media_type: part.source.mimeType as 'application/pdf', + } + } else { + docSource = { + type: 'url', + url: part.source.value, + } + } return { type: 'document', source: docSource, @@ -714,7 +765,7 @@ export class AnthropicTextAdapter< } if (role === 'assistant' && message.toolCalls?.length) { - const contentBlocks: Array = [] + const contentBlocks: Array = [] this.appendThinkingBlocks(contentBlocks, message.thinking) @@ -776,7 +827,7 @@ export class AnthropicTextAdapter< } if (role === 'assistant') { - const contentBlocks: Array = [] + const contentBlocks: Array = [] this.appendThinkingBlocks(contentBlocks, message.thinking) if (Array.isArray(message.content)) { @@ -828,7 +879,7 @@ export class AnthropicTextAdapter< } private appendThinkingBlocks( - contentBlocks: Array, + contentBlocks: Array, thinkingParts: ModelMessage['thinking'], ): void { if (!thinkingParts?.length) return diff --git a/packages/ai-anthropic/src/index.ts b/packages/ai-anthropic/src/index.ts index 468edaf5f..539a7649e 100644 --- a/packages/ai-anthropic/src/index.ts +++ b/packages/ai-anthropic/src/index.ts @@ -19,6 +19,14 @@ export { type AnthropicSummarizeConfig, type AnthropicSummarizeModel, } from './adapters/summarize' + +// Files adapter - upload media to the Anthropic Files API (beta) by file_id +export { + AnthropicFilesAdapter, + createAnthropicFiles, + anthropicFiles, + type AnthropicFilesConfig, +} from './adapters/files' // ============================================================================ // Type Exports // ============================================================================ diff --git a/packages/ai-anthropic/src/text/text-provider-options.ts b/packages/ai-anthropic/src/text/text-provider-options.ts index 9a1811df3..f5b416ee7 100644 --- a/packages/ai-anthropic/src/text/text-provider-options.ts +++ b/packages/ai-anthropic/src/text/text-provider-options.ts @@ -1,15 +1,13 @@ import type { BetaContextManagementConfig, + BetaMessageParam, BetaToolChoiceAny, BetaToolChoiceAuto, BetaToolChoiceTool, } from '@anthropic-ai/sdk/resources/beta/messages/messages' import type { CacheControlEphemeral } from '@anthropic-ai/sdk/resources' import type { AnthropicContainerSkill, AnthropicTool } from '../tools/index' -import type { - MessageParam, - TextBlockParam, -} from '@anthropic-ai/sdk/resources/messages' +import type { TextBlockParam } from '@anthropic-ai/sdk/resources/messages' /** * Per-prompt metadata Anthropic understands on `systemPrompts` entries. @@ -303,7 +301,7 @@ export type ExternalTextProviderOptions = AnthropicContainerOptions & export interface InternalTextProviderOptions extends ExternalTextProviderOptions { model: string - messages: Array + messages: Array /** * The maximum number of tokens to generate before stopping. This parameter only specifies the absolute maximum number of tokens to generate. diff --git a/packages/ai-anthropic/tests/files-source.test.ts b/packages/ai-anthropic/tests/files-source.test.ts new file mode 100644 index 000000000..f176d04b4 --- /dev/null +++ b/packages/ai-anthropic/tests/files-source.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { AnthropicTextAdapter } from '../src/adapters/text' +import type { StreamChunk } from '@tanstack/ai' + +const mocks = vi.hoisted(() => { + const betaMessagesCreate = vi.fn() + const client = { beta: { messages: { create: betaMessagesCreate } } } + return { betaMessagesCreate, client } +}) + +vi.mock('@anthropic-ai/sdk', () => { + const { client } = mocks + class MockAnthropic { + beta = client.beta + constructor(_: { apiKey: string }) {} + } + return { default: MockAnthropic } +}) + +function mockEmptyStream() { + mocks.betaMessagesCreate.mockResolvedValueOnce( + (async function* () { + yield { + type: 'message_delta', + delta: { stop_reason: 'end_turn' }, + usage: { output_tokens: 1 }, + } + yield { type: 'message_stop' } + })(), + ) +} + +async function drain(iterable: AsyncIterable) { + for await (const _ of iterable) { + // consume + } +} + +describe('anthropic file content source', () => { + it('maps an anthropic file handle to a file_id source and sends the Files beta', async () => { + mockEmptyStream() + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-4-1') + + await drain( + chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'describe' }, + { + type: 'image', + source: { + type: 'file', + value: 'file_anthropic_123', + provider: 'anthropic', + }, + }, + ], + }, + ], + }), + ) + + const [payload] = mocks.betaMessagesCreate.mock.calls[0]! + const userMsg = payload.messages.at(-1) + const imageBlock = userMsg.content.find((b: any) => b.type === 'image') + expect(imageBlock.source).toEqual({ + type: 'file', + file_id: 'file_anthropic_123', + }) + expect(payload.betas).toContain('files-api-2025-04-14') + }) + + it('errors when a foreign provider file handle reaches the anthropic adapter', async () => { + mockEmptyStream() + const adapter = new AnthropicTextAdapter({ apiKey: 'k' }, 'claude-opus-4-1') + + const chunks: Array = [] + for await (const chunk of chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'file', + value: 'file-openai-1', + provider: 'openai', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/anthropic/) + } + }) +}) diff --git a/packages/ai-bedrock/src/converse/message-converter.ts b/packages/ai-bedrock/src/converse/message-converter.ts index dfe1c02e3..c75beb70d 100644 --- a/packages/ai-bedrock/src/converse/message-converter.ts +++ b/packages/ai-bedrock/src/converse/message-converter.ts @@ -1,4 +1,8 @@ -import { normalizeSystemPrompts } from '@tanstack/ai' +import { + isFileSource, + normalizeSystemPrompts, + unsupportedFileSourceError, +} from '@tanstack/ai' import type { ContentPart, ContentPartDataSource, @@ -106,6 +110,7 @@ function contentPartToBlock(part: ContentPart, docIndex: number): ContentBlock { if (isImagePart(part)) { const { source } = part + if (isFileSource(source)) throw unsupportedFileSourceError('bedrock') if (!isDataSource(source)) { throw new Error( 'Bedrock Converse requires inline image bytes; URL image sources are not supported.', @@ -121,6 +126,7 @@ function contentPartToBlock(part: ContentPart, docIndex: number): ContentBlock { if (isDocumentPart(part)) { const { source } = part + if (isFileSource(source)) throw unsupportedFileSourceError('bedrock') if (!isDataSource(source)) { throw new Error( 'Bedrock Converse requires inline document bytes; URL document sources are not supported.', diff --git a/packages/ai-event-client/src/index.ts b/packages/ai-event-client/src/index.ts index 7465168be..21dbeb87a 100644 --- a/packages/ai-event-client/src/index.ts +++ b/packages/ai-event-client/src/index.ts @@ -33,7 +33,17 @@ export interface ContentPartUrlSource { mimeType?: string } -export type ContentPartSource = ContentPartDataSource | ContentPartUrlSource +export interface ContentPartFileSource { + type: 'file' + value: string + provider: string + mimeType?: string +} + +export type ContentPartSource = + | ContentPartDataSource + | ContentPartUrlSource + | ContentPartFileSource export interface TextPart { type: 'text' diff --git a/packages/ai-fal/src/adapters/files.ts b/packages/ai-fal/src/adapters/files.ts new file mode 100644 index 000000000..aa782ab7c --- /dev/null +++ b/packages/ai-fal/src/adapters/files.ts @@ -0,0 +1,57 @@ +import { fal } from '@fal-ai/client' +import { + BaseFilesAdapter, + normalizeFileUploadInput, +} from '@tanstack/ai/adapters' +import { configureFalClient } from '../utils/client' +import type { StorageSettings } from '@fal-ai/client' +import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' +import type { FalClientConfig } from '../utils/client' + +export interface FalFilesConfig extends FalClientConfig { + /** + * Optional lifecycle for uploaded objects — one of fal's presets + * (`'1h' | '1d' | '7d' | '30d' | '1y' | 'never' | 'immediate'`) or a number + * of seconds. Applied as the object's `X-Fal-Object-Lifecycle-Preference`. + */ + expiresIn?: StorageSettings['expiresIn'] +} + +/** + * fal Files adapter — uploads media to fal storage via `fal.storage.upload`. + * The handle is a storage URL, so `fileSourceFromHandle(handle)` references it + * as a normal URL (fal endpoints accept it directly). Upload-only: fal storage + * has no retrieval/deletion API, so `get`/`delete` are unavailable. + */ +export class FalFilesAdapter extends BaseFilesAdapter { + readonly name = 'fal' as const + private readonly expiresIn?: StorageSettings['expiresIn'] + + constructor(config?: FalFilesConfig) { + super() + configureFalClient(config) + this.expiresIn = config?.expiresIn + } + + async upload(input: FileUploadInput): Promise { + const { blob, mimeType } = normalizeFileUploadInput(input) + const url = await fal.storage.upload( + blob, + this.expiresIn ? { lifecycle: { expiresIn: this.expiresIn } } : undefined, + ) + return { + id: url, + provider: 'fal', + uri: url, + ...(mimeType ? { mimeType } : {}), + } + } +} + +/** + * Create a fal Files adapter. Reads the API key from `FAL_KEY` unless one is + * provided in `config`. + */ +export function falFiles(config?: FalFilesConfig): FalFilesAdapter { + return new FalFilesAdapter(config) +} diff --git a/packages/ai-fal/src/adapters/video.ts b/packages/ai-fal/src/adapters/video.ts index e2d19e9b0..4fffcdb5b 100644 --- a/packages/ai-fal/src/adapters/video.ts +++ b/packages/ai-fal/src/adapters/video.ts @@ -7,7 +7,10 @@ import { } from '../utils/client' import { buildFalUsage, takeBillableUnits } from '../utils/billing' import { mapVideoSizeToFalFormat } from '../video/video-provider-options' -import { mapImageInputsToFalVideoFields } from '../image/image-inputs' +import { + contentSourceToFalUrl, + mapImageInputsToFalVideoFields, +} from '../image/image-inputs' import type { AudioPart, MediaInputMetadata, @@ -70,17 +73,12 @@ function mapAudioInputsToFalFields( ) } return { - audio_url: - part.source.type === 'url' - ? part.source.value - : `data:${part.source.mimeType};base64,${part.source.value}`, + audio_url: contentSourceToFalUrl(part.source), } } function videoPartToUrl(part: VideoPart): string { - return part.source.type === 'url' - ? part.source.value - : `data:${part.source.mimeType};base64,${part.source.value}` + return contentSourceToFalUrl(part.source) } type FalQueueStatus = 'IN_QUEUE' | 'IN_PROGRESS' | 'COMPLETED' diff --git a/packages/ai-fal/src/image/image-inputs.ts b/packages/ai-fal/src/image/image-inputs.ts index 6196627c6..692cf0237 100644 --- a/packages/ai-fal/src/image/image-inputs.ts +++ b/packages/ai-fal/src/image/image-inputs.ts @@ -1,11 +1,31 @@ +import { assertOwnFileSource, isFileSource } from '@tanstack/ai' import { FAL_IMAGE_FIELD_OVERRIDES } from './generated/image-field-overrides' import type { FalImageFieldName, FalImageFieldOverride, } from './generated/image-field-overrides' -import type { ImagePart, MediaInputMetadata } from '@tanstack/ai' +import type { + ContentPartSource, + ImagePart, + MediaInputMetadata, +} from '@tanstack/ai' import type { FalModel, FalModelInput } from '../model-meta' +/** + * Convert a content source into a URL string for fal's URL-based input fields. + * URL sources pass through; fal storage handles (a storage URL) pass through + * after a provider check; base64 data becomes a `data:;base64,` + * URI which fal endpoints accept on the wire. + */ +export function contentSourceToFalUrl(source: ContentPartSource): string { + if (isFileSource(source)) { + assertOwnFileSource(source, 'fal') + return source.value + } + if (source.type === 'url') return source.value + return `data:${source.mimeType};base64,${source.value}` +} + /** * The image-conditioning fields the mappers may set, narrowed to the ones * that actually exist on the given endpoint's input type. For endpoints @@ -237,10 +257,8 @@ export function mapImageInputsToFalVideoFields( /** * Convert a TanStack ImagePart into a string suitable for fal's URL-based - * input fields. URL sources pass through; data sources are emitted as a - * `data:;base64,` URI which fal endpoints accept on the wire. + * input fields. */ function imagePartToUrl(part: ImagePart): string { - if (part.source.type === 'url') return part.source.value - return `data:${part.source.mimeType};base64,${part.source.value}` + return contentSourceToFalUrl(part.source) } diff --git a/packages/ai-fal/src/index.ts b/packages/ai-fal/src/index.ts index d4a73058f..3b3ed8267 100644 --- a/packages/ai-fal/src/index.ts +++ b/packages/ai-fal/src/index.ts @@ -31,6 +31,12 @@ export { export { FalAudioAdapter, falAudio } from './adapters/audio' +// ============================================================================ +// Files Adapter (storage upload) +// ============================================================================ + +export { FalFilesAdapter, falFiles, type FalFilesConfig } from './adapters/files' + // ============================================================================ // Model Types (from fal.ai's type system) // ============================================================================ diff --git a/packages/ai-fal/tests/content-source-to-fal-url.test.ts b/packages/ai-fal/tests/content-source-to-fal-url.test.ts new file mode 100644 index 000000000..e8dbd01f4 --- /dev/null +++ b/packages/ai-fal/tests/content-source-to-fal-url.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from 'vitest' +import { contentSourceToFalUrl } from '../src/image/image-inputs' + +describe('contentSourceToFalUrl', () => { + it('passes a fal storage handle through as its URL', () => { + expect( + contentSourceToFalUrl({ + type: 'file', + value: 'https://fal.media/files/abc.png', + provider: 'fal', + }), + ).toBe('https://fal.media/files/abc.png') + }) + + it('rejects a file handle issued by another provider', () => { + expect(() => + contentSourceToFalUrl({ + type: 'file', + value: 'file-openai-123', + provider: 'openai', + }), + ).toThrow(/fal/) + }) + + it('passes URL sources through and encodes data sources', () => { + expect( + contentSourceToFalUrl({ type: 'url', value: 'https://x/y.png' }), + ).toBe('https://x/y.png') + expect( + contentSourceToFalUrl({ + type: 'data', + value: 'AAAA', + mimeType: 'image/png', + }), + ).toBe('data:image/png;base64,AAAA') + }) +}) diff --git a/packages/ai-gemini/src/adapters/files.ts b/packages/ai-gemini/src/adapters/files.ts new file mode 100644 index 000000000..0cb9ce3ea --- /dev/null +++ b/packages/ai-gemini/src/adapters/files.ts @@ -0,0 +1,84 @@ +import { + BaseFilesAdapter, + normalizeFileUploadInput, +} from '@tanstack/ai/adapters' +import { createGeminiClient, getGeminiApiKeyFromEnv } from '../utils/client' +import type { File as GeminiFile, GoogleGenAI } from '@google/genai' +import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' +import type { GeminiClientConfig } from '../utils/client' + +export interface GeminiFilesConfig extends GeminiClientConfig {} + +/** + * Gemini Files adapter — uploads media to the Gemini Files API and references + * it by its file URI. Pair with `geminiText()` / `geminiImage()`: reference the + * returned handle via `fileSourceFromHandle(handle)`, which uses the handle URI + * (Gemini fetches it server-side as `fileData.fileUri`). + */ +export class GeminiFilesAdapter extends BaseFilesAdapter { + readonly name = 'gemini' as const + private readonly client: GoogleGenAI + + constructor(config: GeminiFilesConfig) { + super() + this.client = createGeminiClient(config) + } + + async upload(input: FileUploadInput): Promise { + const { blob, mimeType } = normalizeFileUploadInput(input) + const file = await this.client.files.upload({ + file: blob, + ...(mimeType ? { config: { mimeType } } : {}), + }) + return toFileHandle(file) + } + + async get(id: string): Promise { + return toFileHandle(await this.client.files.get({ name: id })) + } + + async delete(id: string): Promise { + await this.client.files.delete({ name: id }) + } +} + +function toFileHandle(file: GeminiFile): FileHandle { + // `name` (e.g. "files/abc-123") is the lifecycle id; `uri` is the URL Gemini + // fetches when the handle is referenced in a message. + if (!file.name) { + throw new Error('gemini: files.upload returned a file without a name') + } + const expiresAt = file.expirationTime + ? Date.parse(file.expirationTime) + : undefined + return { + id: file.name, + provider: 'gemini', + ...(file.uri ? { uri: file.uri } : {}), + ...(file.mimeType ? { mimeType: file.mimeType } : {}), + ...(file.sizeBytes ? { sizeBytes: Number(file.sizeBytes) } : {}), + ...(expiresAt !== undefined && !Number.isNaN(expiresAt) + ? { expiresAt } + : {}), + } +} + +/** + * Create a Gemini Files adapter with an explicit API key. + */ +export function createGeminiFiles( + apiKey: string, + config?: Omit, +): GeminiFilesAdapter { + return new GeminiFilesAdapter({ apiKey, ...config }) +} + +/** + * Create a Gemini Files adapter, reading the API key from `GOOGLE_API_KEY` / + * `GEMINI_API_KEY`. + */ +export function geminiFiles( + config?: Omit, +): GeminiFilesAdapter { + return createGeminiFiles(getGeminiApiKeyFromEnv(), config) +} diff --git a/packages/ai-gemini/src/adapters/image.ts b/packages/ai-gemini/src/adapters/image.ts index 85284d282..e7d9b2205 100644 --- a/packages/ai-gemini/src/adapters/image.ts +++ b/packages/ai-gemini/src/adapters/image.ts @@ -1,4 +1,8 @@ -import { resolveMediaPrompt } from '@tanstack/ai' +import { + assertOwnFileSource, + isFileSource, + resolveMediaPrompt, +} from '@tanstack/ai' import { BaseImageAdapter } from '@tanstack/ai/adapters' import { createGeminiClient, @@ -261,6 +265,11 @@ export class GeminiImageAdapter< }, } } + // A Gemini Files API handle from another provider is a bug — reject it + // before it's passed through as a fileData URI. + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + } // URL sources (public HTTPS, Files API URIs, gs://) pass through as // `fileData` and Gemini fetches them server-side — same as the chat // adapter. Fetching locally and inlining as base64 double-buffers the diff --git a/packages/ai-gemini/src/adapters/text.ts b/packages/ai-gemini/src/adapters/text.ts index 1b7587bb6..aadfab453 100644 --- a/packages/ai-gemini/src/adapters/text.ts +++ b/packages/ai-gemini/src/adapters/text.ts @@ -1,5 +1,10 @@ import { FinishReason } from '@google/genai' -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + assertOwnFileSource, + isFileSource, + normalizeSystemPrompts, +} from '@tanstack/ai' import { toRunErrorRawEvent } from '@tanstack/ai/adapter-internals' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' @@ -662,6 +667,12 @@ export class GeminiTextAdapter< }, } } else { + // File handles (Gemini Files API) and public URLs both pass through as + // `fileData`; Gemini fetches the URI server-side. Reject a handle from + // another provider before it's sent. + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + } // For URL sources, use provided mimeType or fall back to reasonable defaults const defaultMimeType = { image: 'image/jpeg', @@ -766,6 +777,9 @@ export class GeminiTextAdapter< }, }) } else { + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + } const defaultMimeType = { image: 'image/jpeg', audio: 'audio/mp3', diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts index 4fc2897fd..18e5157ed 100644 --- a/packages/ai-gemini/src/adapters/video.ts +++ b/packages/ai-gemini/src/adapters/video.ts @@ -2,7 +2,11 @@ import { GenerateVideosOperation, VideoGenerationReferenceType, } from '@google/genai' -import { resolveMediaPrompt } from '@tanstack/ai' +import { + isFileSource, + resolveMediaPrompt, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters' import { arrayBufferToBase64 } from '@tanstack/ai-utils' import { createGeminiClient, getGeminiApiKeyFromEnv } from '../utils' @@ -93,6 +97,14 @@ async function imagePartToVeoImage( mimeType: part.source.mimeType || 'image/png', } } + if (isFileSource(part.source)) { + // Veo's predict API accepts only inline bytes or a gs:// reference — there's + // no way to reference a Files API handle here. + throw unsupportedFileSourceError( + 'gemini', + 'for Veo video generation, which needs inline image bytes or a gs:// reference — pass a data: URI or gs:// URL', + ) + } const url = part.source.value if (url.startsWith('gs://')) { return { diff --git a/packages/ai-gemini/src/experimental/text-interactions/adapter.ts b/packages/ai-gemini/src/experimental/text-interactions/adapter.ts index 9d1de8ffb..2173f2874 100644 --- a/packages/ai-gemini/src/experimental/text-interactions/adapter.ts +++ b/packages/ai-gemini/src/experimental/text-interactions/adapter.ts @@ -1,4 +1,4 @@ -import { EventType } from '@tanstack/ai' +import { EventType, assertOwnFileSource, isFileSource } from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { parse as parsePartialJSON } from 'partial-json' import { @@ -728,6 +728,11 @@ function contentPartToBlock(part: ContentPart): ContentBlock { if (part.type === 'text') { return { type: 'text', text: part.content } } + // A file handle from another provider is a bug; a Gemini handle maps to the + // `uri` field (isData stays false), same as a public URL. + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, 'gemini') + } const isData = part.source.type === 'data' switch (part.type) { case 'image': { diff --git a/packages/ai-gemini/src/index.ts b/packages/ai-gemini/src/index.ts index 245f892cb..49611ed55 100644 --- a/packages/ai-gemini/src/index.ts +++ b/packages/ai-gemini/src/index.ts @@ -19,6 +19,14 @@ export { type GeminiSummarizeModel, } from './adapters/summarize' +// Files adapter - upload media to the Gemini Files API and reference by file URI +export { + GeminiFilesAdapter, + createGeminiFiles, + geminiFiles, + type GeminiFilesConfig, +} from './adapters/files' + // Image adapter export { GeminiImageAdapter, diff --git a/packages/ai-grok/src/adapters/image.ts b/packages/ai-grok/src/adapters/image.ts index 50bd38e51..f26ea0a2a 100644 --- a/packages/ai-grok/src/adapters/image.ts +++ b/packages/ai-grok/src/adapters/image.ts @@ -1,5 +1,9 @@ import OpenAI from 'openai' -import { resolveMediaPrompt } from '@tanstack/ai' +import { + isFileSource, + resolveMediaPrompt, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseImageAdapter } from '@tanstack/ai/adapters' import { toRunErrorPayload } from '@tanstack/ai/adapter-internals' import { buildImagesUsage } from '@tanstack/openai-base' @@ -62,6 +66,7 @@ function imagineSizeParams(size: string | undefined): { * sources become base64 data URIs. */ function imagePartToUrl(part: ImagePart): string { + if (isFileSource(part.source)) throw unsupportedFileSourceError('grok') if (part.source.type === 'url') return part.source.value return `data:${part.source.mimeType};base64,${part.source.value}` } diff --git a/packages/ai-grok/src/adapters/video.ts b/packages/ai-grok/src/adapters/video.ts index 21807360e..02954bb49 100644 --- a/packages/ai-grok/src/adapters/video.ts +++ b/packages/ai-grok/src/adapters/video.ts @@ -1,4 +1,8 @@ -import { resolveMediaPrompt } from '@tanstack/ai' +import { + isFileSource, + resolveMediaPrompt, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters' import { toRunErrorPayload } from '@tanstack/ai/adapter-internals' import { getGrokApiKeyFromEnv, withGrokDefaults } from '../utils/client' @@ -67,6 +71,7 @@ interface GrokVideoStatusResponse { * sources become base64 data URIs. */ function imagePartToUrl(part: ImagePart): string { + if (isFileSource(part.source)) throw unsupportedFileSourceError('grok') if (part.source.type === 'url') return part.source.value return `data:${part.source.mimeType};base64,${part.source.value}` } diff --git a/packages/ai-mistral/src/adapters/text.ts b/packages/ai-mistral/src/adapters/text.ts index 9760a106b..6e50e67d1 100644 --- a/packages/ai-mistral/src/adapters/text.ts +++ b/packages/ai-mistral/src/adapters/text.ts @@ -1,3 +1,4 @@ +import { isFileSource, unsupportedFileSourceError } from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { convertToolsToProviderFormat } from '../tools/tool-converter' import { @@ -1007,6 +1008,9 @@ export class MistralTextAdapter< } if (part.type === 'image') { + if (isFileSource(part.source)) { + throw unsupportedFileSourceError('mistral') + } const imageMetadata = part.metadata as MistralImageMetadata | undefined const imageValue = part.source.value const imageUrl = diff --git a/packages/ai-ollama/src/adapters/text.ts b/packages/ai-ollama/src/adapters/text.ts index b7ae351f2..75c920ef7 100644 --- a/packages/ai-ollama/src/adapters/text.ts +++ b/packages/ai-ollama/src/adapters/text.ts @@ -1,4 +1,9 @@ -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + isFileSource, + normalizeSystemPrompts, + unsupportedFileSourceError, +} from '@tanstack/ai' import { toRunErrorPayload, toRunErrorRawEvent, @@ -469,11 +474,10 @@ export class OllamaTextAdapter extends BaseTextAdapter< if (part.type === 'text') { textContent += part.content } else if (part.type === 'image') { - if (part.source.type === 'data') { - images.push(part.source.value) - } else { - images.push(part.source.value) + if (isFileSource(part.source)) { + throw unsupportedFileSourceError('ollama') } + images.push(part.source.value) } } } else { diff --git a/packages/ai-openai/src/adapters/files.ts b/packages/ai-openai/src/adapters/files.ts new file mode 100644 index 000000000..81e76029e --- /dev/null +++ b/packages/ai-openai/src/adapters/files.ts @@ -0,0 +1,86 @@ +import { OpenAI, toFile } from 'openai' +import { + BaseFilesAdapter, + normalizeFileUploadInput, +} from '@tanstack/ai/adapters' +import { getOpenAIApiKeyFromEnv } from '../utils/client' +import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' +import type { FileObject, FilePurpose } from 'openai/resources/files' +import type { OpenAIClientConfig } from '../utils/client' + +export interface OpenAIFilesConfig extends OpenAIClientConfig { + /** + * Default `purpose` for uploads. Files uploaded for vision/document input to + * the Responses API use `'user_data'` (the flexible default). Override per + * upload need — e.g. `'vision'` — via this config. + * @default 'user_data' + */ + purpose?: FilePurpose +} + +/** + * OpenAI Files adapter — uploads media to the OpenAI Files API and references + * it by `file_id`. Pair with `openaiText()` (Responses API): reference the + * returned handle in a message via `fileSourceFromHandle(handle)`. + */ +export class OpenAIFilesAdapter extends BaseFilesAdapter { + readonly name = 'openai' as const + protected client: OpenAI + private readonly purpose: FilePurpose + + constructor(config: OpenAIFilesConfig) { + super() + const { purpose, ...clientOptions } = config + this.client = new OpenAI(clientOptions) + this.purpose = purpose ?? 'user_data' + } + + async upload(input: FileUploadInput): Promise { + const { blob, mimeType, filename } = normalizeFileUploadInput(input) + const file = await toFile(blob, filename, { + ...(mimeType ? { type: mimeType } : {}), + }) + const result = await this.client.files.create({ + file, + purpose: this.purpose, + }) + return toFileHandle(result) + } + + async get(id: string): Promise { + return toFileHandle(await this.client.files.retrieve(id)) + } + + async delete(id: string): Promise { + await this.client.files.delete(id) + } +} + +function toFileHandle(file: FileObject): FileHandle { + return { + id: file.id, + provider: 'openai', + sizeBytes: file.bytes, + filename: file.filename, + ...(file.expires_at ? { expiresAt: file.expires_at * 1000 } : {}), + } +} + +/** + * Create an OpenAI Files adapter with an explicit API key. + */ +export function createOpenaiFiles( + apiKey: string, + config?: Omit, +): OpenAIFilesAdapter { + return new OpenAIFilesAdapter({ apiKey, ...config }) +} + +/** + * Create an OpenAI Files adapter, reading the API key from `OPENAI_API_KEY`. + */ +export function openaiFiles( + config?: Omit, +): OpenAIFilesAdapter { + return createOpenaiFiles(getOpenAIApiKeyFromEnv(), config) +} diff --git a/packages/ai-openai/src/image/image-input-to-file.ts b/packages/ai-openai/src/image/image-input-to-file.ts index 77c3f2a30..28b864cad 100644 --- a/packages/ai-openai/src/image/image-input-to-file.ts +++ b/packages/ai-openai/src/image/image-input-to-file.ts @@ -1,4 +1,5 @@ import { base64ToArrayBuffer } from '@tanstack/ai-utils' +import { isFileSource, unsupportedFileSourceError } from '@tanstack/ai' import type { ImagePart, MediaInputMetadata } from '@tanstack/ai' const DEFAULT_MIME = 'image/png' @@ -44,6 +45,15 @@ export async function imagePartToFile( ): Promise { ensureFileSupport() + if (isFileSource(part.source)) { + // The edits / Sora input_reference endpoints are multipart and require the + // actual bytes; there's no "reference an uploaded file_id" option here. + throw unsupportedFileSourceError( + 'openai', + 'on the images/edits + Sora input_reference endpoints, which require uploaded bytes — pass a data: URI or inline image data', + ) + } + if (part.source.type === 'data') { const mimeType = part.source.mimeType || DEFAULT_MIME const bytes = base64ToArrayBuffer(part.source.value) diff --git a/packages/ai-openai/src/index.ts b/packages/ai-openai/src/index.ts index fdeb79d57..bfc435930 100644 --- a/packages/ai-openai/src/index.ts +++ b/packages/ai-openai/src/index.ts @@ -78,6 +78,14 @@ export { } from './adapters/transcription' export type { OpenAITranscriptionProviderOptions } from './audio/transcription-provider-options' +// Files adapter - upload media to the OpenAI Files API and reference by file_id +export { + OpenAIFilesAdapter, + createOpenaiFiles, + openaiFiles, + type OpenAIFilesConfig, +} from './adapters/files' + // ============================================================================ // Type Exports // ============================================================================ diff --git a/packages/ai-openai/tests/files-source.test.ts b/packages/ai-openai/tests/files-source.test.ts new file mode 100644 index 000000000..88c923846 --- /dev/null +++ b/packages/ai-openai/tests/files-source.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { OpenAITextAdapter } from '../src/adapters/text' +import type { StreamChunk } from '@tanstack/ai' + +function mockResponsesStream(): AsyncIterable> { + return { + // eslint-disable-next-line @typescript-eslint/require-await + async *[Symbol.asyncIterator]() { + yield { + type: 'response.created', + response: { id: 'r1', model: 'gpt-4o-mini', status: 'in_progress' }, + } + yield { + type: 'response.completed', + response: { + id: 'r1', + model: 'gpt-4o-mini', + status: 'completed', + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + }, + } + }, + } +} + +function withMockClient(create: ReturnType) { + const adapter = new OpenAITextAdapter({ apiKey: 'k' }, 'gpt-4o-mini') + ;(adapter as unknown as { client: unknown }).client = { + responses: { create }, + } + return adapter +} + +async function drain(iterable: AsyncIterable) { + for await (const _ of iterable) { + // consume + } +} + +describe('openai file content source', () => { + it('maps an openai file handle to input_image.file_id on the Responses API', async () => { + const create = vi.fn().mockResolvedValueOnce(mockResponsesStream()) + const adapter = withMockClient(create) + + await drain( + chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'look' }, + { + type: 'image', + source: { + type: 'file', + value: 'file-openai-abc', + provider: 'openai', + }, + }, + ], + }, + ], + }), + ) + + const [payload] = create.mock.calls[0]! + const userItem = payload.input.find( + (item: any) => item.type === 'message' && item.role === 'user', + ) + const imageContent = userItem.content.find( + (c: any) => c.type === 'input_image', + ) + expect(imageContent.file_id).toBe('file-openai-abc') + expect(imageContent.image_url).toBeUndefined() + }) + + it('errors when a foreign provider file handle reaches the openai adapter', async () => { + const create = vi.fn().mockResolvedValueOnce(mockResponsesStream()) + const adapter = withMockClient(create) + + const chunks: Array = [] + for await (const chunk of chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'file', + value: 'files/gemini-xyz', + provider: 'gemini', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/openai/) + } + }) +}) diff --git a/packages/ai-openrouter/src/adapters/image.ts b/packages/ai-openrouter/src/adapters/image.ts index daf0ecb9d..e17cb5824 100644 --- a/packages/ai-openrouter/src/adapters/image.ts +++ b/packages/ai-openrouter/src/adapters/image.ts @@ -1,5 +1,9 @@ import { OpenRouter } from '@openrouter/sdk' -import { resolveMediaPrompt } from '@tanstack/ai' +import { + isFileSource, + resolveMediaPrompt, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseImageAdapter } from '@tanstack/ai/adapters' import { getOpenRouterApiKeyFromEnv, @@ -51,6 +55,7 @@ const SIZE_TO_ASPECT_RATIO: Record = { * base64 data URIs. */ function imagePartToUrl(part: ImagePart): string { + if (isFileSource(part.source)) throw unsupportedFileSourceError('openrouter') if (part.source.type === 'url') return part.source.value return `data:${part.source.mimeType};base64,${part.source.value}` } diff --git a/packages/ai-openrouter/src/adapters/responses-text.ts b/packages/ai-openrouter/src/adapters/responses-text.ts index 782ac081a..f95cccf33 100644 --- a/packages/ai-openrouter/src/adapters/responses-text.ts +++ b/packages/ai-openrouter/src/adapters/responses-text.ts @@ -1,5 +1,10 @@ import { OpenRouter } from '@openrouter/sdk' -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + isFileSource, + normalizeSystemPrompts, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { toRunErrorPayload, @@ -1680,6 +1685,9 @@ export class OpenRouterResponsesTextAdapter< protected convertContentPartToInput( part: ContentPart, ): ResponsesInputContent { + if ('source' in part && isFileSource(part.source)) { + throw unsupportedFileSourceError(this.name) + } switch (part.type) { case 'text': return { diff --git a/packages/ai-openrouter/src/adapters/text.ts b/packages/ai-openrouter/src/adapters/text.ts index 09df05b35..b7c9cd43c 100644 --- a/packages/ai-openrouter/src/adapters/text.ts +++ b/packages/ai-openrouter/src/adapters/text.ts @@ -1,5 +1,10 @@ import { OpenRouter } from '@openrouter/sdk' -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + isFileSource, + normalizeSystemPrompts, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { toRunErrorPayload, @@ -1297,6 +1302,9 @@ export class OpenRouterTextAdapter< /** OpenRouter content-part converter (camelCase imageUrl/inputAudio/videoUrl). */ protected convertContentPart(part: ContentPart): ChatContentItems | null { + if ('source' in part && isFileSource(part.source)) { + throw unsupportedFileSourceError(this.name) + } switch (part.type) { case 'text': return { type: 'text', text: part.content } diff --git a/packages/ai/src/activities/files/adapter.ts b/packages/ai/src/activities/files/adapter.ts new file mode 100644 index 000000000..5d62ad0a4 --- /dev/null +++ b/packages/ai/src/activities/files/adapter.ts @@ -0,0 +1,106 @@ +/** + * Files Adapter + * + * Base class and interface for the `files` activity — a provider's native Files + * API (upload a media asset once, reference it later by the returned handle + * instead of re-sending base64 or a public URL each request). + * + * Providers with a native surface expose a factory (`openaiFiles()`, + * `anthropicFiles()`, `geminiFiles()`, `falFiles()`). `upload` is required; + * `get`/`delete` are optional because not every provider has a lifecycle API + * (fal's storage is upload-only). + */ + +import { base64ToArrayBuffer } from '@tanstack/ai-utils' + +/** + * Input to {@link FilesAdapter.upload}. Either a `Blob` (memory-efficient, + * preferred for large assets) or base64 `data` plus its `mimeType`. + */ +export type FileUploadInput = + | Blob + | { + /** Base64-encoded file bytes. */ + data: string + /** MIME type of the bytes (e.g. `'image/png'`, `'application/pdf'`). */ + mimeType: string + /** Optional filename hint sent to providers that accept one. */ + filename?: string + } + +/** + * A provider-issued file handle returned by {@link FilesAdapter.upload} / + * {@link FilesAdapter.get}. Reference it in a message via a `{ type: 'file' }` + * content source — use {@link fileSourceFromHandle} to build one. + */ +export interface FileHandle { + /** + * Provider handle used for lifecycle operations (`get`/`delete`): the + * OpenAI/Anthropic `file_id`, the Gemini file resource name (`files/...`), or + * the fal storage URL. + */ + id: string + /** The provider that issued the handle (`'openai'`, `'gemini'`, ...). */ + provider: string + /** + * The handle's URL form when the provider exposes one (Gemini file URI, fal + * storage URL). For providers whose handle is an opaque id (OpenAI, + * Anthropic) this is `undefined`. + */ + uri?: string + /** MIME type reported by the provider (or echoed from the upload input). */ + mimeType?: string + /** File size in bytes when the provider reports it. */ + sizeBytes?: number + /** Expiry as epoch milliseconds when the handle is scheduled to expire. */ + expiresAt?: number + /** Original filename when the provider reports it. */ + filename?: string +} + +/** + * The `files` adapter contract. `upload` is required; `get`/`delete` are + * optional and present only when the provider has a lifecycle API. + */ +export interface FilesAdapter { + readonly kind: 'files' + readonly name: string + upload: (input: FileUploadInput) => Promise + get?: (id: string) => Promise + delete?: (id: string) => Promise +} + +export type AnyFilesAdapter = FilesAdapter + +/** + * Normalize a {@link FileUploadInput} to a `Blob` (plus best-effort MIME / + * filename) so provider adapters can hand it straight to their SDK. A `Blob` + * input passes through; base64 `{ data }` is decoded to bytes. Shared so the + * four provider files adapters don't each re-implement the decode. + */ +export function normalizeFileUploadInput(input: FileUploadInput): { + blob: Blob + mimeType?: string + filename?: string +} { + if (input instanceof Blob) { + return { blob: input, mimeType: input.type || undefined } + } + const bytes = base64ToArrayBuffer(input.data) + return { + blob: new Blob([bytes], { type: input.mimeType }), + mimeType: input.mimeType, + filename: input.filename, + } +} + +/** + * Abstract base for provider files adapters. Subclasses set `name`, implement + * `upload`, and optionally implement `get`/`delete`. + */ +export abstract class BaseFilesAdapter implements FilesAdapter { + readonly kind = 'files' as const + abstract readonly name: string + + abstract upload(input: FileUploadInput): Promise +} diff --git a/packages/ai/src/activities/files/index.ts b/packages/ai/src/activities/files/index.ts new file mode 100644 index 000000000..5a098fa78 --- /dev/null +++ b/packages/ai/src/activities/files/index.ts @@ -0,0 +1,94 @@ +/** + * Files Activity + * + * Dispatch functions for provider Files APIs. Each takes `{ adapter, ... }` and + * calls the adapter method directly (mirrors the other activity dispatchers). + * `get`/`delete` are optional on the adapter; the dispatchers throw a clear + * error when the selected provider has no lifecycle API. + */ + +import type { ContentPartFileSource } from '../../types' +import type { AnyFilesAdapter, FileHandle, FileUploadInput } from './adapter' + +/** The adapter kind this activity handles */ +export const kind = 'files' as const + +/** + * Upload a file to a provider's Files API and return its handle. + * + * @example + * ```ts + * const files = openaiFiles() + * const handle = await uploadFile({ adapter: files, input: { data, mimeType: 'image/png' } }) + * ``` + */ +export async function uploadFile(options: { + adapter: TAdapter & { kind: typeof kind } + input: FileUploadInput +}): Promise { + return options.adapter.upload(options.input) +} + +/** + * Fetch metadata for a previously uploaded file by its handle id. + * + * @throws if the provider's files adapter has no `get` (e.g. fal storage). + */ +export async function getFile(options: { + adapter: TAdapter & { kind: typeof kind } + id: string +}): Promise { + const { adapter, id } = options + if (!adapter.get) { + throw new Error( + `${adapter.name}: files adapter does not support get() — this provider ` + + `has no file-retrieval API.`, + ) + } + return adapter.get(id) +} + +/** + * Delete a previously uploaded file by its handle id. + * + * @throws if the provider's files adapter has no `delete` (e.g. fal storage). + */ +export async function deleteFile(options: { + adapter: TAdapter & { kind: typeof kind } + id: string +}): Promise { + const { adapter, id } = options + if (!adapter.delete) { + throw new Error( + `${adapter.name}: files adapter does not support delete() — this ` + + `provider has no file-deletion API.`, + ) + } + return adapter.delete(id) +} + +/** + * Build a `{ type: 'file' }` content source from an uploaded {@link FileHandle}, + * for use in a chat message (image/audio/document part `source`). + * + * Picks the right `value`: the handle URL when the provider exposes one + * (Gemini/fal), otherwise the opaque id (OpenAI/Anthropic). + * + * @example + * ```ts + * const handle = await uploadFile({ adapter: openaiFiles(), input }) + * messages.push({ role: 'user', content: [ + * { type: 'image', source: fileSourceFromHandle(handle) }, + * ] }) + * ``` + */ +export function fileSourceFromHandle( + handle: FileHandle, +): ContentPartFileSource { + return { + type: 'file', + value: handle.uri ?? handle.id, + provider: handle.provider, + ...(handle.mimeType ? { mimeType: handle.mimeType } : {}), + } +} diff --git a/packages/ai/src/activities/index.ts b/packages/ai/src/activities/index.ts index 07fdbe73a..56e683eb1 100644 --- a/packages/ai/src/activities/index.ts +++ b/packages/ai/src/activities/index.ts @@ -21,6 +21,7 @@ import type { AnyAudioAdapter } from './generateAudio/adapter' import type { AnyVideoAdapter } from './generateVideo/adapter' import type { AnyTTSAdapter } from './generateSpeech/adapter' import type { AnyTranscriptionAdapter } from './generateTranscription/adapter' +import type { AnyFilesAdapter } from './files/adapter' // =========================== // Chat Activity @@ -170,11 +171,32 @@ export { type AnyTranscriptionAdapter, } from './generateTranscription/adapter' +// =========================== +// Files Activity +// =========================== + +export { + kind as filesKind, + uploadFile, + getFile, + deleteFile, + fileSourceFromHandle, +} from './files/index' + +export { + BaseFilesAdapter, + normalizeFileUploadInput, + type FilesAdapter, + type AnyFilesAdapter, + type FileHandle, + type FileUploadInput, +} from './files/adapter' + // =========================== // Adapter Union Types // =========================== -/** Union of all adapter types that can be passed to chat() */ +/** Union of all adapter types across every activity kind */ export type AIAdapter = | AnyTextAdapter | AnySummarizeAdapter @@ -183,6 +205,7 @@ export type AIAdapter = | AnyVideoAdapter | AnyTTSAdapter | AnyTranscriptionAdapter + | AnyFilesAdapter /** Union type of all adapter kinds */ export type AdapterKind = @@ -193,3 +216,4 @@ export type AdapterKind = | 'video' | 'tts' | 'transcription' + | 'files' diff --git a/packages/ai/src/client.ts b/packages/ai/src/client.ts index a677e92ea..4656708bb 100644 --- a/packages/ai/src/client.ts +++ b/packages/ai/src/client.ts @@ -303,6 +303,7 @@ export type { AudioPart, ContentPart, ContentPartDataSource, + ContentPartFileSource, ContentPartSource, ContentPartUrlSource, CustomEvent, diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 20e5e7cb7..1b6659770 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -8,6 +8,10 @@ export { getVideoJobStatus, generateSpeech, generateTranscription, + uploadFile, + getFile, + deleteFile, + fileSourceFromHandle, } from './activities/index' // Create options functions - for pre-defining typed configurations @@ -36,6 +40,10 @@ export type { TranscriptionAdapter, AnyVideoAdapter, VideoAdapter, + FilesAdapter, + AnyFilesAdapter, + FileHandle, + FileUploadInput, } from './activities/index' // Tool definition @@ -395,6 +403,11 @@ export { isContentPartArray, normalizeToolResult, } from './utilities/tool-result' +export { + assertOwnFileSource, + isFileSource, + unsupportedFileSourceError, +} from './utilities/content-source' export { getProviderExecutedMetadata, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index af1a20059..3979e351d 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -244,13 +244,52 @@ export interface ContentPartUrlSource { mimeType?: string } +/** + * Source specification for a provider-issued file handle (Files API). + * + * The media is uploaded once via a `files` adapter (`openaiFiles()`, + * `anthropicFiles()`, `geminiFiles()`, `falFiles()`) and referenced here by the + * returned handle instead of re-sending base64 or a public URL each request. + * + * A handle only routes to the provider that issued it — the `provider` field is + * validated at map time, and adapters throw if it doesn't match. The `value` is + * the provider's opaque id (OpenAI/Anthropic `file_id`) or handle URL (Gemini + * file URI, fal storage URL); use {@link fileSourceFromHandle} to build one from + * a {@link FileHandle} without worrying about the id-vs-uri distinction. + */ +export interface ContentPartFileSource { + /** + * Indicates this references a provider-issued file handle. + */ + type: 'file' + /** + * The provider handle: an OpenAI/Anthropic `file_id`, a Gemini file URI, or a + * fal storage URL. + */ + value: string + /** + * The provider that issued the handle. A handle is only valid for its issuer; + * passing (e.g.) an OpenAI `file-...` id to a Gemini adapter is an error. + */ + provider: string + /** + * Optional MIME type hint for cases where the provider can't infer it. + */ + mimeType?: string +} + /** * Source specification for multimodal content. - * Discriminated union supporting both inline data (base64) and URL-based content. + * Discriminated union supporting inline data (base64), URL-based content, and + * provider-issued file handles. * - For 'data' sources: mimeType is required * - For 'url' sources: mimeType is optional + * - For 'file' sources: a provider-issued handle plus its issuing `provider` */ -export type ContentPartSource = ContentPartDataSource | ContentPartUrlSource +export type ContentPartSource = + | ContentPartDataSource + | ContentPartUrlSource + | ContentPartFileSource /** * Image content part for multimodal messages. diff --git a/packages/ai/src/utilities/content-source.ts b/packages/ai/src/utilities/content-source.ts new file mode 100644 index 000000000..f61ad3432 --- /dev/null +++ b/packages/ai/src/utilities/content-source.ts @@ -0,0 +1,57 @@ +import type { ContentPartFileSource, ContentPartSource } from '../types' + +/** + * Narrow a {@link ContentPartSource} to the provider-file-handle arm. + * + * Every adapter that maps a content part's `source` onto a provider wire format + * must handle `{ type: 'file' }` explicitly — either mapping it to the provider's + * native file-reference field (issuers) or rejecting it (everyone else). Using + * this guard keeps that branch consistent across the ~dozen adapter packages. + */ +export function isFileSource( + source: ContentPartSource, +): source is ContentPartFileSource { + return source.type === 'file' +} + +/** + * Assert that a file source's handle was issued by `providerName`. A provider + * file handle is only valid for the provider that created it (an OpenAI + * `file-...` id sent to Gemini is a bug), so issuer adapters call this before + * mapping the handle onto their wire format. + * + * @throws if `source.provider` doesn't match `providerName`. + */ +export function assertOwnFileSource( + source: ContentPartFileSource, + providerName: string, +): void { + if (source.provider !== providerName) { + throw new Error( + `${providerName}: file source references a handle issued by ` + + `"${source.provider}" — a provider file handle only works with the ` + + `provider that created it. Upload the file with ${providerName}Files() ` + + `and reference that handle, or pass a data/url source instead.`, + ) + } +} + +/** + * Build the standard error a non-issuer adapter throws when it encounters a + * `{ type: 'file' }` source it can't consume — either because the provider has + * no file-handle input surface, or because the endpoint requires raw bytes + * (image edits, Veo) rather than a reference. + * + * @param detail Optional context appended to the message (e.g. a modality or + * endpoint name, or a pointer to the adapter that does support handles). + */ +export function unsupportedFileSourceError( + providerName: string, + detail?: string, +): Error { + return new Error( + `${providerName} does not support provider file-handle sources ` + + `({ type: 'file' })${detail ? ` ${detail}` : ''}. Pass a data or url ` + + `source, or upload via the provider's files adapter where supported.`, + ) +} diff --git a/packages/ai/src/utilities/tool-result.ts b/packages/ai/src/utilities/tool-result.ts index 330c29be1..5a18039ec 100644 --- a/packages/ai/src/utilities/tool-result.ts +++ b/packages/ai/src/utilities/tool-result.ts @@ -11,7 +11,7 @@ const CONTENT_PART_TYPES = new Set([ /** * Structural check for a single `ContentPart`. A text part must carry a string * `content`; every other modality must carry a `source` with `type` of - * `'url' | 'data'` and a string `value`. + * `'url' | 'data' | 'file'` and a string `value`. */ export function isContentPart(value: unknown): value is ContentPart { if (typeof value !== 'object' || value === null) return false @@ -30,6 +30,8 @@ export function isContentPart(value: unknown): value is ContentPart { // sources don't. Requiring it here keeps the runtime guard consistent with // the type and avoids emitting `data:undefined;base64,...` downstream. if (src.type === 'data') return typeof src.mimeType === 'string' + // `file` sources reference a provider-issued handle and must name their issuer. + if (src.type === 'file') return typeof src.provider === 'string' return src.type === 'url' } diff --git a/packages/ai/tests/files-source.test.ts b/packages/ai/tests/files-source.test.ts new file mode 100644 index 000000000..fc9aaecd8 --- /dev/null +++ b/packages/ai/tests/files-source.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it } from 'vitest' +import { + assertOwnFileSource, + deleteFile, + fileSourceFromHandle, + getFile, + isContentPart, + isFileSource, + unsupportedFileSourceError, + uploadFile, +} from '../src/index' +import { normalizeFileUploadInput } from '../src/activities/files/adapter' +import type { ContentPartSource } from '../src/types' +import type { FileHandle, FilesAdapter } from '../src/activities/files/adapter' + +const fileSource: ContentPartSource = { + type: 'file', + value: 'file-abc', + provider: 'openai', +} + +describe('file content source helpers', () => { + it('isFileSource narrows only the file arm', () => { + expect(isFileSource(fileSource)).toBe(true) + expect(isFileSource({ type: 'url', value: 'https://x/y' })).toBe(false) + expect( + isFileSource({ type: 'data', value: 'AAAA', mimeType: 'image/png' }), + ).toBe(false) + }) + + it('assertOwnFileSource passes on match and throws on mismatch', () => { + expect(() => assertOwnFileSource(fileSource, 'openai')).not.toThrow() + expect(() => assertOwnFileSource(fileSource, 'gemini')).toThrow(/openai/) + }) + + it('unsupportedFileSourceError includes provider and detail', () => { + const err = unsupportedFileSourceError('mistral', 'on this endpoint') + expect(err.message).toContain('mistral') + expect(err.message).toContain('on this endpoint') + }) + + it('fileSourceFromHandle prefers uri (Gemini/fal), else id (OpenAI/Anthropic)', () => { + const opaque: FileHandle = { id: 'file-abc', provider: 'openai' } + expect(fileSourceFromHandle(opaque)).toEqual({ + type: 'file', + value: 'file-abc', + provider: 'openai', + }) + + const withUri: FileHandle = { + id: 'files/xyz', + provider: 'gemini', + uri: 'https://generativelanguage.googleapis.com/v1/files/xyz', + mimeType: 'image/png', + } + expect(fileSourceFromHandle(withUri)).toEqual({ + type: 'file', + value: 'https://generativelanguage.googleapis.com/v1/files/xyz', + provider: 'gemini', + mimeType: 'image/png', + }) + }) + + it('isContentPart accepts a valid file source and rejects one missing provider', () => { + expect( + isContentPart({ type: 'image', source: fileSource }), + ).toBe(true) + expect( + isContentPart({ + type: 'image', + source: { type: 'file', value: 'file-abc' }, + }), + ).toBe(false) + }) +}) + +describe('normalizeFileUploadInput', () => { + it('passes a Blob through and decodes base64 input', async () => { + const blob = new Blob(['hi'], { type: 'text/plain' }) + expect(normalizeFileUploadInput(blob).blob).toBe(blob) + + const fromBase64 = normalizeFileUploadInput({ + data: 'aGVsbG8=', // "hello" + mimeType: 'text/plain', + filename: 'greeting.txt', + }) + expect(fromBase64.mimeType).toBe('text/plain') + expect(fromBase64.filename).toBe('greeting.txt') + expect(await fromBase64.blob.text()).toBe('hello') + }) +}) + +describe('files activity dispatch', () => { + const uploadOnly: FilesAdapter = { + kind: 'files', + name: 'fal', + upload: async () => ({ id: 'https://cdn/x', provider: 'fal' }), + } + const full: FilesAdapter = { + kind: 'files', + name: 'openai', + upload: async () => ({ id: 'file-1', provider: 'openai' }), + get: async (id) => ({ id, provider: 'openai' }), + delete: async () => {}, + } + + it('uploadFile returns the handle', async () => { + const handle = await uploadFile({ + adapter: uploadOnly, + input: new Blob(['x']), + }) + expect(handle).toEqual({ id: 'https://cdn/x', provider: 'fal' }) + }) + + it('getFile / deleteFile throw when the adapter has no lifecycle API', async () => { + await expect(getFile({ adapter: uploadOnly, id: 'x' })).rejects.toThrow( + /does not support get/, + ) + await expect( + deleteFile({ adapter: uploadOnly, id: 'x' }), + ).rejects.toThrow(/does not support delete/) + }) + + it('getFile / deleteFile call through when supported', async () => { + expect(await getFile({ adapter: full, id: 'file-1' })).toEqual({ + id: 'file-1', + provider: 'openai', + }) + await expect( + deleteFile({ adapter: full, id: 'file-1' }), + ).resolves.toBeUndefined() + }) +}) diff --git a/packages/openai-base/src/adapters/chat-completions-text.ts b/packages/openai-base/src/adapters/chat-completions-text.ts index 72ea19ca7..68751777a 100644 --- a/packages/openai-base/src/adapters/chat-completions-text.ts +++ b/packages/openai-base/src/adapters/chat-completions-text.ts @@ -1,4 +1,9 @@ -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + isFileSource, + normalizeSystemPrompts, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { toRunErrorPayload, @@ -1312,6 +1317,15 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< | { detail?: 'auto' | 'low' | 'high' } | undefined + if (isFileSource(part.source)) { + // The Chat Completions API references images only by URL/data URI, not + // by an uploaded file_id — point callers at the Responses adapter. + throw unsupportedFileSourceError( + this.name, + 'on the Chat Completions API — use the Responses adapter (e.g. openaiText) to reference an uploaded file by file_id', + ) + } + // For base64 data, construct a data URI using the mimeType from source. // Default to a generic octet-stream MIME if the source didn't provide // one — interpolating `undefined` into the URI ("data:undefined;base64, diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index e467ac5e6..dc5b6fe02 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -1,4 +1,9 @@ -import { EventType, normalizeSystemPrompts } from '@tanstack/ai' +import { + EventType, + assertOwnFileSource, + isFileSource, + normalizeSystemPrompts, +} from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { toRunErrorPayload, @@ -1799,6 +1804,14 @@ export abstract class OpenAIBaseResponsesTextAdapter< const imageMetadata = part.metadata as | { detail?: 'auto' | 'low' | 'high' } | undefined + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + return { + type: 'input_image', + file_id: part.source.value, + detail: imageMetadata?.detail || 'auto', + } + } if (part.source.type === 'url') { return { type: 'input_image', @@ -1822,6 +1835,13 @@ export abstract class OpenAIBaseResponsesTextAdapter< } } case 'audio': { + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + return { + type: 'input_file', + file_id: part.source.value, + } + } if (part.source.type === 'url') { return { type: 'input_file', @@ -1842,8 +1862,19 @@ export abstract class OpenAIBaseResponsesTextAdapter< } } + case 'document': { + // A document uploaded via the Files API is referenced by `file_id`; + // inline document bytes/URLs aren't accepted on this path. + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, this.name) + return { + type: 'input_file', + file_id: part.source.value, + } + } + throw new Error(`Unsupported content part type: ${part.type}`) + } case 'video': - case 'document': default: // OpenAI Responses API doesn't accept native video/document parts on // this path — surface as explicit unsupported error so callers see From 50e0da0efa6c2fcacebb18a2b95c68a808ded50f Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Wed, 8 Jul 2026 05:50:13 +0000 Subject: [PATCH 2/5] ci: apply automated fixes --- packages/ai-anthropic/src/adapters/files.ts | 5 ++++- packages/ai-anthropic/src/adapters/text.ts | 4 +--- packages/ai-fal/src/index.ts | 6 +++++- packages/ai/tests/files-source.test.ts | 10 ++++------ 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/ai-anthropic/src/adapters/files.ts b/packages/ai-anthropic/src/adapters/files.ts index b0b5f6bcd..b897efd58 100644 --- a/packages/ai-anthropic/src/adapters/files.ts +++ b/packages/ai-anthropic/src/adapters/files.ts @@ -3,7 +3,10 @@ import { BaseFilesAdapter, normalizeFileUploadInput, } from '@tanstack/ai/adapters' -import { createAnthropicClient, getAnthropicApiKeyFromEnv } from '../utils/client' +import { + createAnthropicClient, + getAnthropicApiKeyFromEnv, +} from '../utils/client' import type Anthropic_SDK from '@anthropic-ai/sdk' import type { FileMetadata } from '@anthropic-ai/sdk/resources/beta/files' import type { FileHandle, FileUploadInput } from '@tanstack/ai/adapters' diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index 0c77e316a..d87099183 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -157,9 +157,7 @@ function buildServerToolResultBlock( * True when any message carries a provider file-handle source, so the request * must send the Files API beta header. */ -export function messagesHaveFileSource( - messages: Array, -): boolean { +export function messagesHaveFileSource(messages: Array): boolean { return messages.some( (message) => Array.isArray(message.content) && diff --git a/packages/ai-fal/src/index.ts b/packages/ai-fal/src/index.ts index 3b3ed8267..16ab46fe8 100644 --- a/packages/ai-fal/src/index.ts +++ b/packages/ai-fal/src/index.ts @@ -35,7 +35,11 @@ export { FalAudioAdapter, falAudio } from './adapters/audio' // Files Adapter (storage upload) // ============================================================================ -export { FalFilesAdapter, falFiles, type FalFilesConfig } from './adapters/files' +export { + FalFilesAdapter, + falFiles, + type FalFilesConfig, +} from './adapters/files' // ============================================================================ // Model Types (from fal.ai's type system) diff --git a/packages/ai/tests/files-source.test.ts b/packages/ai/tests/files-source.test.ts index fc9aaecd8..d8e41cd28 100644 --- a/packages/ai/tests/files-source.test.ts +++ b/packages/ai/tests/files-source.test.ts @@ -62,9 +62,7 @@ describe('file content source helpers', () => { }) it('isContentPart accepts a valid file source and rejects one missing provider', () => { - expect( - isContentPart({ type: 'image', source: fileSource }), - ).toBe(true) + expect(isContentPart({ type: 'image', source: fileSource })).toBe(true) expect( isContentPart({ type: 'image', @@ -116,9 +114,9 @@ describe('files activity dispatch', () => { await expect(getFile({ adapter: uploadOnly, id: 'x' })).rejects.toThrow( /does not support get/, ) - await expect( - deleteFile({ adapter: uploadOnly, id: 'x' }), - ).rejects.toThrow(/does not support delete/) + await expect(deleteFile({ adapter: uploadOnly, id: 'x' })).rejects.toThrow( + /does not support delete/, + ) }) it('getFile / deleteFile call through when supported', async () => { From 346881a42916e73ced4dbf26d692904fa203992e Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:01:00 +1000 Subject: [PATCH 3/5] example(ts-react-media): use the Files API for reference-image inputs + Nitro note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upload reference images (Gemini) and image-to-video start frames (fal) once via the Files API (`geminiFiles()` / `falFiles()`) and reference them by handle, instead of re-sending the base64 payload inline on every generation request. Bump the example's `nitro` to `latest` (3.0.260610-beta) — older Nitro rejected `@google/genai`'s resumable upload (explicit `Content-Length` on a Blob body) with "invalid content-length header", surfaced as "fetch failed". Document that runtime requirement in the Files API guide. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/advanced/files-api.md | 9 +++ .../src/components/ImageGenerator.tsx | 3 +- .../src/lib/server-functions.ts | 76 ++++++++++++++++--- 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/docs/advanced/files-api.md b/docs/advanced/files-api.md index fcf5d260c..86c74e54b 100644 --- a/docs/advanced/files-api.md +++ b/docs/advanced/files-api.md @@ -44,6 +44,15 @@ const handle = await openaiFiles().upload({ - `uri` — the handle's URL form when the provider exposes one (Gemini file URI, fal storage URL); `undefined` for OpenAI/Anthropic, whose handles are opaque ids. - `expiresAt` — epoch milliseconds, when the provider schedules the handle to expire. +> **Runtime note (Gemini upload).** `geminiFiles().upload()` uses `@google/genai`'s +> resumable upload, which sets an explicit `Content-Length` header on a `Blob`-body +> request. Some server runtimes reject that with `fetch failed` / +> `InvalidArgumentError: invalid content-length header`. On **TanStack Start / Nitro** +> this fails on older Nitro (observed on `nitro@3.0.1-alpha.2`) and works on current +> Nitro (verified on `nitro@3.0.260610-beta`) — upgrade Nitro if you hit it. Native +> Node (and the production `node-server` build) are unaffected. OpenAI, Anthropic, and +> fal uploads use different transports and don't exercise this path. + ### get and delete Providers with a lifecycle API expose `get()` and `delete()`: diff --git a/examples/ts-react-media/src/components/ImageGenerator.tsx b/examples/ts-react-media/src/components/ImageGenerator.tsx index d1af51297..b825a32b3 100644 --- a/examples/ts-react-media/src/components/ImageGenerator.tsx +++ b/examples/ts-react-media/src/components/ImageGenerator.tsx @@ -182,7 +182,8 @@ export default function ImageGenerator({ Sent as image prompt parts with role "reference" — - accepted by the Gemini multimodal models, xAI Imagine and Seedream + accepted by the Gemini multimodal models (uploaded once via the + Gemini Files API), xAI Imagine and Seedream
diff --git a/examples/ts-react-media/src/lib/server-functions.ts b/examples/ts-react-media/src/lib/server-functions.ts index 64432beff..f653d7ad1 100644 --- a/examples/ts-react-media/src/lib/server-functions.ts +++ b/examples/ts-react-media/src/lib/server-functions.ts @@ -1,6 +1,6 @@ import { createServerFn } from '@tanstack/react-start' -import { falImage, falVideo } from '@tanstack/ai-fal' -import { geminiImage, geminiVideo } from '@tanstack/ai-gemini' +import { falFiles, falImage, falVideo } from '@tanstack/ai-fal' +import { geminiFiles, geminiImage, geminiVideo } from '@tanstack/ai-gemini' import { grokImage, grokVideo } from '@tanstack/ai-grok' import { BYTEPLUS_VIDEO_MODELS, @@ -12,12 +12,13 @@ import { supportsReferenceMedia, } from '@tanstack/ai-byteplus' import { + fileSourceFromHandle, generateImage, generateVideo, toServerSentEventsResponse, } from '@tanstack/ai' -import type { StreamChunk } from '@tanstack/ai' +import type { FilesAdapter, StreamChunk } from '@tanstack/ai' import type { BytePlusVideoModel, BytePlusVideoModelOrString, @@ -110,6 +111,31 @@ function asImageToVideoPrompt( return narrowed } +/** + * Upload each inline (base64 `data`) image input to the provider's Files API and + * swap in a `{ type: 'file' }` handle. A reference image / start frame is then + * uploaded once via the tree-shakeable files adapter (`geminiFiles()` / + * `falFiles()`) instead of being re-sent inline as base64 on the generation + * request — the memory-safe path for large inputs. URL and already-uploaded + * sources pass through untouched. + */ +async function uploadInlineImageInputs( + prompt: string | Array>, + files: FilesAdapter, +): Promise>> { + if (typeof prompt === 'string') return prompt + return Promise.all( + prompt.map(async (part) => { + if (part.type !== 'image' || part.source.type !== 'data') return part + const handle = await files.upload({ + data: part.source.value, + mimeType: part.source.mimeType, + }) + return { ...part, source: fileSourceFromHandle(handle) } + }), + ) +} + /** * Poll cadence for the streamed video lifecycle. The server holds the request * open and polls the provider itself, so this is the rate at which @@ -196,9 +222,14 @@ export const generateImageFn = createServerFn({ method: 'POST' }) }) } case 'gemini-3.1-flash-image-preview': { + // Reference images are uploaded once via the Gemini Files API and + // referenced by handle (fileData.fileUri) rather than inlined as base64. return generateImage({ adapter: geminiImage('gemini-3.1-flash-image-preview'), - prompt: asImagePrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImagePrompt(data.prompt), + geminiFiles(), + ), numberOfImages: 1, size: '16:9_4K', }) @@ -206,7 +237,10 @@ export const generateImageFn = createServerFn({ method: 'POST' }) case 'gemini-3-pro-image-preview': { return generateImage({ adapter: geminiImage('gemini-3-pro-image-preview'), - prompt: asImagePrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImagePrompt(data.prompt), + geminiFiles(), + ), numberOfImages: 1, size: '16:9_4K', }) @@ -280,7 +314,9 @@ interface VideoRequest { * browser's `useGenerateVideo` reads job id, status and result off these * chunks instead of running its own timer. */ -function videoStreamForModel(data: VideoRequest): AsyncIterable { +async function videoStreamForModel( + data: VideoRequest, +): Promise> { // Image-to-video models receive the start frame as a prompt part // (role: 'start_frame') — the fal adapter routes it to the endpoint's // start-image field. Text-to-video models take the text prompt only. @@ -362,13 +398,18 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { size: '16:9_2160p', }) } - // Image-to-video models + // Image-to-video models. The start frame is uploaded once to fal storage + // via the Files API (`falFiles()`) and referenced by its storage-URL + // handle, instead of being inlined as a base64 data: URI on the request. case 'fal-ai/kling-video/v3/pro/image-to-video': { return generateVideo({ stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/kling-video/v3/pro/image-to-video'), - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falFiles(), + ), modelOptions: { generate_audio: true, duration: '5', @@ -380,7 +421,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/veo3.1/image-to-video'), - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falFiles(), + ), size: '16:9_1080p', modelOptions: { duration: '4s', @@ -392,7 +436,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('xai/grok-imagine-video/image-to-video'), - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falFiles(), + ), size: '16:9_720p', modelOptions: { duration: 5, @@ -417,7 +464,10 @@ function videoStreamForModel(data: VideoRequest): AsyncIterable { stream: true, pollingInterval: VIDEO_POLL_INTERVAL_MS, adapter: falVideo('fal-ai/ltx-2.3/image-to-video/fast'), - prompt: asImageToVideoPrompt(data.prompt), + prompt: await uploadInlineImageInputs( + asImageToVideoPrompt(data.prompt), + falFiles(), + ), size: '16:9_2160p', }) } @@ -483,7 +533,9 @@ export const generateVideoFn = createServerFn({ method: 'POST' }) // before any stream exists, which surfaces as a plain server-function error // (the hook reports it through `error`) rather than a stream that opens only // to fail. - .handler(({ data }) => toServerSentEventsResponse(videoStreamForModel(data))) + .handler(async ({ data }) => + toServerSentEventsResponse(await videoStreamForModel(data)), + ) // ============================================================================ // Seedance Studio — BytePlus ModelArk direct (ARK_API_KEY, server-side only) From 7f0f9d54f46a44ca3f49f6e9e3b88ce6154c71fe Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:41:41 +1000 Subject: [PATCH 4/5] review: complete the file-source sweep, gate openai-base subclasses, thread provider-literal handle types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-review completion pass for the Files API PR (rebased onto main): - ai-byteplus (landed on main after this branch): file-source guards in text/image/video adapters + tests — a handle would previously have been sent to Ark as a URL. - Gemini Interactions video path: add the missing assertOwnFileSource guard (the sibling text-interactions adapter already had it) + tests. - openai-base Responses: new `supportsFileIdInput` gate (true only for OpenAI itself) so Grok/Bedrock/compatible subclasses reject file sources instead of inheriting the file_id mapping; chat-completions error copy no longer points non-OpenAI providers at openaiText. Tests for both. - Provider-literal handle types: FileHandle/FilesAdapter thread each adapter's name literal through uploadFile; getFile/deleteFile accept the handle itself, making cross-provider lifecycle calls compile errors. fileSourceFromHandle + FileHandle now also exported from the browser-safe @tanstack/ai/client entry. - Docs: fix broken openaiText import in multimodal-content.md, document the uploadFile/getFile/deleteFile dispatchers, correct the fal get/delete and explicit-key claims; kiira green. Skills: adapter-configuration §7 + chat-experience/media-generation notes. - Tests: gemini files-source mapping/rejection, OpenAI Responses document- arm behavior, toFileHandle normalizers (openai/gemini/fal), handle-object lifecycle dispatch, and an aimock e2e (file-source-wire) covering the round-trip + cross-provider rejection end-to-end. - Polish: unsupportedFileSourceError detail no longer contradicted by the generic tail, ollama uses this.name, fal expiresIn !== undefined, comment rot fixes; changeset updated (adds ai-byteplus patch). --- .changeset/native-files-api-support.md | 4 +- docs/advanced/files-api.md | 56 +++++--- docs/advanced/multimodal-content.md | 10 +- docs/config.json | 4 +- .../src/lib/server-functions.ts | 7 +- packages/ai-anthropic/src/adapters/files.ts | 8 +- packages/ai-byteplus/src/adapters/image.ts | 7 +- packages/ai-byteplus/src/adapters/text.ts | 10 +- packages/ai-byteplus/src/adapters/video.ts | 7 +- .../ai-byteplus/tests/files-source.test.ts | 99 ++++++++++++++ packages/ai-fal/src/adapters/files.ts | 8 +- packages/ai-fal/tests/files-adapter.test.ts | 69 ++++++++++ packages/ai-gemini/src/adapters/files.ts | 8 +- packages/ai-gemini/src/adapters/video.ts | 6 + .../ai-gemini/tests/files-adapter.test.ts | 83 ++++++++++++ packages/ai-gemini/tests/files-source.test.ts | 124 ++++++++++++++++++ packages/ai-grok/tests/files-source.test.ts | 83 ++++++++++++ packages/ai-ollama/src/adapters/text.ts | 2 +- packages/ai-openai/src/adapters/files.ts | 8 +- packages/ai-openai/src/adapters/text.ts | 4 + .../ai-openai/tests/files-adapter.test.ts | 75 +++++++++++ packages/ai-openai/tests/files-source.test.ts | 70 ++++++++++ .../ai-core/adapter-configuration/SKILL.md | 51 +++++++ .../skills/ai-core/chat-experience/SKILL.md | 8 ++ .../skills/ai-core/media-generation/SKILL.md | 8 ++ packages/ai/src/activities/files/adapter.ts | 46 ++++--- packages/ai/src/activities/files/index.ts | 56 +++++--- packages/ai/src/client.ts | 7 + packages/ai/src/types.ts | 8 +- packages/ai/src/utilities/content-source.ts | 11 +- packages/ai/tests/files-source.test.ts | 26 ++++ .../src/adapters/chat-completions-text.ts | 13 +- .../src/adapters/responses-text.ts | 30 ++++- testing/e2e/src/routeTree.gen.ts | 21 +++ .../e2e/src/routes/api.file-source-wire.ts | 98 ++++++++++++++ testing/e2e/tests/file-source-wire.spec.ts | 97 ++++++++++++++ 36 files changed, 1131 insertions(+), 101 deletions(-) create mode 100644 packages/ai-byteplus/tests/files-source.test.ts create mode 100644 packages/ai-fal/tests/files-adapter.test.ts create mode 100644 packages/ai-gemini/tests/files-adapter.test.ts create mode 100644 packages/ai-gemini/tests/files-source.test.ts create mode 100644 packages/ai-grok/tests/files-source.test.ts create mode 100644 packages/ai-openai/tests/files-adapter.test.ts create mode 100644 testing/e2e/src/routes/api.file-source-wire.ts create mode 100644 testing/e2e/tests/file-source-wire.spec.ts diff --git a/.changeset/native-files-api-support.md b/.changeset/native-files-api-support.md index 361362ba5..3b355fdb5 100644 --- a/.changeset/native-files-api-support.md +++ b/.changeset/native-files-api-support.md @@ -11,6 +11,7 @@ '@tanstack/ai-openrouter': patch '@tanstack/ai-ollama': patch '@tanstack/ai-bedrock': patch +'@tanstack/ai-byteplus': patch --- feat(ai): native Files API support across providers (upload adapters + `file` content source) @@ -19,4 +20,5 @@ Adds first-class support for provider **Files / storage APIs** so callers can up - **New tree-shakeable `files` adapter kind** — `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, and `falFiles()`. Each exposes `upload()`, and (where the provider has a lifecycle API) `get()` / `delete()`. Drive them with the new `uploadFile()` / `getFile()` / `deleteFile()` activity functions. fal is upload-only. - **New `{ type: 'file' }` arm on `ContentPartSource`** — reference an uploaded handle in a chat message. Adapters map it to the right wire field: OpenAI (Responses) `input_image`/`input_file` `file_id`, Anthropic `file_id` message source (with the `files-api-2025-04-14` beta), Gemini `fileData.fileUri`, fal storage URL passthrough. Use `fileSourceFromHandle(handle)` to build the source from an uploaded `FileHandle`. -- **Runtime provider routing** — a file handle only routes to the provider that issued it; adapters throw a clear error on a cross-provider handle, and providers/endpoints that can't consume a handle (image edits, Veo, Chat Completions images, Bedrock, Mistral, Grok, OpenRouter, Ollama) throw a clear "unsupported file source" error instead of silently mis-mapping. +- **Runtime provider routing** — a file handle only routes to the provider that issued it; adapters throw a clear error on a cross-provider handle, and providers/endpoints that can't consume a handle (image edits, Veo, Chat Completions images, Bedrock, Mistral, Grok, OpenRouter, Ollama, BytePlus) throw a clear "unsupported file source" error instead of silently mis-mapping. Grok/Bedrock adapters built on the shared OpenAI Responses base are gated too (`supportsFileIdInput`), so an inherited `file_id` mapping can't leak to providers without a Files API. +- **Provider-literal typed handles** — `FileHandle<'openai'>` etc. flow from each files adapter through `uploadFile()`, and `getFile()`/`deleteFile()` accept the handle itself, so cross-provider lifecycle calls fail at compile time. `fileSourceFromHandle` and `FileHandle` are also exported from the browser-safe `@tanstack/ai/client` entry. diff --git a/docs/advanced/files-api.md b/docs/advanced/files-api.md index 86c74e54b..d23f87ca9 100644 --- a/docs/advanced/files-api.md +++ b/docs/advanced/files-api.md @@ -17,25 +17,30 @@ TanStack AI exposes this as a tree-shakeable **`files` adapter** per provider, p ## Files adapters -Each provider with a native surface has a factory: `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, and `falFiles()`. They read the same API-key env var as the provider's other adapters, or accept an explicit key. +Each provider with a native surface has a factory: `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, and `falFiles()`. They read the same API-key env var as the provider's other adapters; to pass a key explicitly, use the `create*Files(apiKey)` variants (`createOpenaiFiles`, `createAnthropicFiles`, `createGeminiFiles`) — `falFiles(config)` takes its key in the config object. ```typescript -import { openaiFiles } from '@tanstack/ai-openai' +import { createOpenaiFiles, openaiFiles } from '@tanstack/ai-openai' import { geminiFiles } from '@tanstack/ai-gemini' import { anthropicFiles } from '@tanstack/ai-anthropic' import { falFiles } from '@tanstack/ai-fal' const files = openaiFiles() // reads OPENAI_API_KEY +const filesWithKey = createOpenaiFiles('sk-your-key') // explicit key ``` -### upload +### uploadFile -`upload()` accepts a `Blob` (memory-efficient — preferred for large assets) or `{ data, mimeType }` where `data` is base64. It returns a `FileHandle`: +Drive an adapter with the `uploadFile()` activity function. It accepts a `Blob` (memory-efficient — preferred for large assets) or `{ data, mimeType }` where `data` is base64, and returns a `FileHandle`: ```typescript -const handle = await openaiFiles().upload({ - data: pdfBase64, - mimeType: 'application/pdf', +import { uploadFile } from '@tanstack/ai' +import { openaiFiles } from '@tanstack/ai-openai' +import { pdfBase64 } from './pdf-data' + +const handle = await uploadFile({ + adapter: openaiFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, }) // handle: { id, provider, uri?, mimeType?, sizeBytes?, expiresAt?, filename? } ``` @@ -53,16 +58,26 @@ const handle = await openaiFiles().upload({ > Node (and the production `node-server` build) are unaffected. OpenAI, Anthropic, and > fal uploads use different transports and don't exercise this path. -### get and delete +### getFile and deleteFile -Providers with a lifecycle API expose `get()` and `delete()`: +Providers with a lifecycle API support `getFile()` and `deleteFile()`. Both accept the handle itself (preferred — the handle's provider type rejects a foreign provider's handle at compile time) or its raw `id`: ```typescript -const meta = await openaiFiles().get(handle.id) -await openaiFiles().delete(handle.id) +import { deleteFile, getFile, uploadFile } from '@tanstack/ai' +import { openaiFiles } from '@tanstack/ai-openai' +import { pdfBase64 } from './pdf-data' + +const files = openaiFiles() +const handle = await uploadFile({ + adapter: files, + input: { data: pdfBase64, mimeType: 'application/pdf' }, +}) + +const meta = await getFile({ adapter: files, id: handle }) +await deleteFile({ adapter: files, id: handle }) ``` -> fal storage is **upload-only** — `falFiles()` has no `get` / `delete`, and calling them throws a clear error. +> fal storage is **upload-only** — `falFiles()` defines no `get` / `delete`, and calling `getFile()` / `deleteFile()` with it throws a clear error. ## Referencing a handle in a message @@ -71,15 +86,14 @@ Use `fileSourceFromHandle(handle)` to turn a `FileHandle` into a `{ type: 'file' ### Server: upload + reference ```typescript -import { chat, fileSourceFromHandle } from '@tanstack/ai' -import { anthropicText } from '@tanstack/ai-anthropic' -import { anthropicFiles } from '@tanstack/ai-anthropic' +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { anthropicFiles, anthropicText } from '@tanstack/ai-anthropic' export async function askAboutPdf(pdfBase64: string, request: string) { // Upload once; reuse the handle across turns. - const handle = await anthropicFiles().upload({ - data: pdfBase64, - mimeType: 'application/pdf', + const handle = await uploadFile({ + adapter: anthropicFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, }) return chat({ @@ -99,11 +113,11 @@ export async function askAboutPdf(pdfBase64: string, request: string) { ### Client: reuse a handle across requests -Upload happens server-side (it needs the provider key), so the client works with the returned handle. Persist `{ id, provider, uri, mimeType }` and rebuild the source on each turn: +Upload happens server-side (it needs the provider key), so the client works with the returned handle. Persist `{ id, provider, uri, mimeType }` and rebuild the source on each turn. `fileSourceFromHandle` and `FileHandle` are exported from the browser-safe `@tanstack/ai/client` entry, so this doesn't pull the server bundle into the client: ```typescript -import { fileSourceFromHandle } from '@tanstack/ai' -import type { FileHandle } from '@tanstack/ai' +import { fileSourceFromHandle } from '@tanstack/ai/client' +import type { FileHandle } from '@tanstack/ai/client' // `handle` was returned by your server's upload endpoint and stored client-side. function imageMessage(handle: FileHandle, prompt: string) { diff --git a/docs/advanced/multimodal-content.md b/docs/advanced/multimodal-content.md index 030cd899f..c02f5fc89 100644 --- a/docs/advanced/multimodal-content.md +++ b/docs/advanced/multimodal-content.md @@ -263,11 +263,15 @@ const imagePart = { Use `type: 'file'` to reference media you uploaded once via a provider's [Files API](./files-api.md) — the provider stores the bytes and you pass a lightweight handle instead of re-sending base64 or a public URL every request. A handle only works with the provider that issued it, so the `provider` field is required and validated at request time. ```typescript -import { openaiFiles } from '@tanstack/ai-openai' -import { openaiText, chat, fileSourceFromHandle } from '@tanstack/ai' +import { openaiFiles, openaiText } from '@tanstack/ai-openai' +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { pdfBase64 } from './pdf-data' // Upload once... -const handle = await openaiFiles().upload({ data: pdfBase64, mimeType: 'application/pdf' }) +const handle = await uploadFile({ + adapter: openaiFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, +}) // ...then reference the handle by id in as many requests as you like. for await (const chunk of chat({ diff --git a/docs/config.json b/docs/config.json index 9ca2ff725..544ea5c63 100644 --- a/docs/config.json +++ b/docs/config.json @@ -637,12 +637,12 @@ "label": "Multimodal Content", "to": "advanced/multimodal-content", "addedAt": "2026-04-15", - "updatedAt": "2026-07-08" + "updatedAt": "2026-08-07" }, { "label": "Files API", "to": "advanced/files-api", - "addedAt": "2026-07-08" + "addedAt": "2026-08-07" }, { "label": "Per-Model Type Safety", diff --git a/examples/ts-react-media/src/lib/server-functions.ts b/examples/ts-react-media/src/lib/server-functions.ts index f653d7ad1..c8a162234 100644 --- a/examples/ts-react-media/src/lib/server-functions.ts +++ b/examples/ts-react-media/src/lib/server-functions.ts @@ -16,6 +16,7 @@ import { generateImage, generateVideo, toServerSentEventsResponse, + uploadFile, } from '@tanstack/ai' import type { FilesAdapter, StreamChunk } from '@tanstack/ai' @@ -127,9 +128,9 @@ async function uploadInlineImageInputs( return Promise.all( prompt.map(async (part) => { if (part.type !== 'image' || part.source.type !== 'data') return part - const handle = await files.upload({ - data: part.source.value, - mimeType: part.source.mimeType, + const handle = await uploadFile({ + adapter: files, + input: { data: part.source.value, mimeType: part.source.mimeType }, }) return { ...part, source: fileSourceFromHandle(handle) } }), diff --git a/packages/ai-anthropic/src/adapters/files.ts b/packages/ai-anthropic/src/adapters/files.ts index b897efd58..ba0c4a8cf 100644 --- a/packages/ai-anthropic/src/adapters/files.ts +++ b/packages/ai-anthropic/src/adapters/files.ts @@ -22,7 +22,7 @@ export interface AnthropicFilesConfig extends AnthropicClientConfig {} * references it by `file_id`. Pair with `anthropicText()`: reference the * returned handle in an image/document message via `fileSourceFromHandle`. */ -export class AnthropicFilesAdapter extends BaseFilesAdapter { +export class AnthropicFilesAdapter extends BaseFilesAdapter<'anthropic'> { readonly name = 'anthropic' as const private readonly client: Anthropic_SDK @@ -31,7 +31,7 @@ export class AnthropicFilesAdapter extends BaseFilesAdapter { this.client = createAnthropicClient(config) } - async upload(input: FileUploadInput): Promise { + async upload(input: FileUploadInput): Promise> { const { blob, mimeType, filename } = normalizeFileUploadInput(input) const file = await toFile(blob, filename, { ...(mimeType ? { type: mimeType } : {}), @@ -43,7 +43,7 @@ export class AnthropicFilesAdapter extends BaseFilesAdapter { return toFileHandle(result) } - async get(id: string): Promise { + async get(id: string): Promise> { const result = await this.client.beta.files.retrieveMetadata(id, { betas: [FILES_API_BETA], }) @@ -55,7 +55,7 @@ export class AnthropicFilesAdapter extends BaseFilesAdapter { } } -function toFileHandle(file: FileMetadata): FileHandle { +function toFileHandle(file: FileMetadata): FileHandle<'anthropic'> { return { id: file.id, provider: 'anthropic', diff --git a/packages/ai-byteplus/src/adapters/image.ts b/packages/ai-byteplus/src/adapters/image.ts index 76e2d52b3..3e5368887 100644 --- a/packages/ai-byteplus/src/adapters/image.ts +++ b/packages/ai-byteplus/src/adapters/image.ts @@ -1,4 +1,8 @@ -import { resolveMediaPrompt } from '@tanstack/ai' +import { + isFileSource, + resolveMediaPrompt, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseImageAdapter } from '@tanstack/ai/adapters' import { toRunErrorPayload } from '@tanstack/ai/adapter-internals' import { generateId } from '@tanstack/ai-utils' @@ -69,6 +73,7 @@ const SUPPORTED_INPUT_ROLES: ReadonlySet = new Set([ */ function imagePartToImageRef(part: ImagePart): string { const { source } = part + if (isFileSource(source)) throw unsupportedFileSourceError('byteplus') if (source.type === 'url') return source.value if (source.value.startsWith('data:')) return source.value return `data:${source.mimeType.toLowerCase()};base64,${source.value}` diff --git a/packages/ai-byteplus/src/adapters/text.ts b/packages/ai-byteplus/src/adapters/text.ts index d9d53684e..b11e14489 100644 --- a/packages/ai-byteplus/src/adapters/text.ts +++ b/packages/ai-byteplus/src/adapters/text.ts @@ -1,5 +1,9 @@ import OpenAI from 'openai' -import { EventType } from '@tanstack/ai' +import { + EventType, + isFileSource, + unsupportedFileSourceError, +} from '@tanstack/ai' import { OpenAIBaseChatCompletionsTextAdapter } from '@tanstack/openai-base' import { generateId } from '@tanstack/ai-utils' import { @@ -262,6 +266,9 @@ export class BytePlusTextAdapter< if (part.type === 'audio') { const metadata = part.metadata as BytePlusAudioMetadata | undefined + if (isFileSource(part.source)) { + throw unsupportedFileSourceError('byteplus') + } // Ark takes audio either by URL or as inline base64 with an explicit // container format; unlike images there is no data-URI form. if (part.source.type === 'url') { @@ -439,6 +446,7 @@ function asChatContentPart( * inline base64 becomes a `data:` URI. */ function toUrlOrDataUri(source: ContentPartSource): string { + if (isFileSource(source)) throw unsupportedFileSourceError('byteplus') if (source.type !== 'data' || source.value.startsWith('data:')) { return source.value } diff --git a/packages/ai-byteplus/src/adapters/video.ts b/packages/ai-byteplus/src/adapters/video.ts index 58b2186d5..554dda3cd 100644 --- a/packages/ai-byteplus/src/adapters/video.ts +++ b/packages/ai-byteplus/src/adapters/video.ts @@ -1,4 +1,8 @@ -import { resolveMediaPrompt } from '@tanstack/ai' +import { + isFileSource, + resolveMediaPrompt, + unsupportedFileSourceError, +} from '@tanstack/ai' import { BaseVideoAdapter, snapToDurationOption } from '@tanstack/ai/adapters' import { toRunErrorPayload } from '@tanstack/ai/adapter-internals' import { @@ -76,6 +80,7 @@ function mediaPartToUrl( | AudioPart, ): string { const { source } = part + if (isFileSource(source)) throw unsupportedFileSourceError('byteplus') if (source.type === 'url') return source.value if (source.value.startsWith('data:')) return source.value return `data:${source.mimeType.toLowerCase()};base64,${source.value}` diff --git a/packages/ai-byteplus/tests/files-source.test.ts b/packages/ai-byteplus/tests/files-source.test.ts new file mode 100644 index 000000000..706c5bb97 --- /dev/null +++ b/packages/ai-byteplus/tests/files-source.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { createBytePlusText } from '../src/adapters/text' +import { BYTEPLUS_CHAT_MODELS } from '../src/model-meta' +import type { StreamChunk } from '@tanstack/ai' + +// Stub the OpenAI SDK so constructing an adapter never opens a real network +// handle (same pattern as text.test.ts). +vi.mock('openai', () => { + return { + default: class { + chat = { + completions: { + create: vi.fn(), + }, + } + }, + } +}) + +// The guard throws while the request body is being built, so the mocked +// `create` is never reached — it exists only to fail loudly if a file source +// ever leaks through to a network call. +function adapterWithMockClient() { + const adapter = createBytePlusText(BYTEPLUS_CHAT_MODELS[0], 'test-key') + ;(adapter as any).client = { chat: { completions: { create: vi.fn() } } } + return adapter +} + +async function collectChunks( + iterable: AsyncIterable, +): Promise> { + const chunks: Array = [] + for await (const chunk of iterable) { + chunks.push(chunk) + } + return chunks +} + +describe('byteplus file content source', () => { + it('rejects a foreign provider file handle instead of sending it as a URL', async () => { + const chunks = await collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'file', + value: 'file-openai-abc', + provider: 'openai', + }, + }, + ], + }, + ], + }), + ) + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/byteplus/) + expect(runError.message).toMatch(/file/) + } + }) + + it('rejects a byteplus-marked file source — the provider has no Files API', async () => { + const chunks = await collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: [ + { + role: 'user', + content: [ + { + type: 'video', + source: { + type: 'file', + value: 'https://example.com/some-handle', + provider: 'byteplus', + }, + }, + ], + }, + ], + }), + ) + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/byteplus/) + } + }) +}) diff --git a/packages/ai-fal/src/adapters/files.ts b/packages/ai-fal/src/adapters/files.ts index aa782ab7c..32bbcdc6c 100644 --- a/packages/ai-fal/src/adapters/files.ts +++ b/packages/ai-fal/src/adapters/files.ts @@ -23,7 +23,7 @@ export interface FalFilesConfig extends FalClientConfig { * as a normal URL (fal endpoints accept it directly). Upload-only: fal storage * has no retrieval/deletion API, so `get`/`delete` are unavailable. */ -export class FalFilesAdapter extends BaseFilesAdapter { +export class FalFilesAdapter extends BaseFilesAdapter<'fal'> { readonly name = 'fal' as const private readonly expiresIn?: StorageSettings['expiresIn'] @@ -33,11 +33,13 @@ export class FalFilesAdapter extends BaseFilesAdapter { this.expiresIn = config?.expiresIn } - async upload(input: FileUploadInput): Promise { + async upload(input: FileUploadInput): Promise> { const { blob, mimeType } = normalizeFileUploadInput(input) const url = await fal.storage.upload( blob, - this.expiresIn ? { lifecycle: { expiresIn: this.expiresIn } } : undefined, + this.expiresIn !== undefined + ? { lifecycle: { expiresIn: this.expiresIn } } + : undefined, ) return { id: url, diff --git a/packages/ai-fal/tests/files-adapter.test.ts b/packages/ai-fal/tests/files-adapter.test.ts new file mode 100644 index 000000000..1c0dd921e --- /dev/null +++ b/packages/ai-fal/tests/files-adapter.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it, vi } from 'vitest' +import { falFiles } from '../src/adapters/files' +import type { FilesAdapter } from '@tanstack/ai/adapters' + +const mocks = vi.hoisted(() => { + return { + storageUpload: vi.fn(), + config: vi.fn(), + } +}) + +vi.mock('@fal-ai/client', () => { + return { + fal: { + config: mocks.config, + storage: { upload: mocks.storageUpload }, + }, + } +}) + +describe('fal files adapter', () => { + it('returns the storage URL as both id and uri', async () => { + mocks.storageUpload.mockResolvedValueOnce('https://fal.media/files/x.png') + const files = falFiles({ apiKey: 'k' }) + + const handle = await files.upload({ data: 'AAAA', mimeType: 'image/png' }) + + expect(handle).toEqual({ + id: 'https://fal.media/files/x.png', + provider: 'fal', + uri: 'https://fal.media/files/x.png', + mimeType: 'image/png', + }) + }) + + it('passes the configured lifecycle through, including 0 seconds', async () => { + mocks.storageUpload.mockResolvedValue('https://fal.media/files/x.png') + + await falFiles({ apiKey: 'k', expiresIn: '7d' }).upload({ + data: 'AAAA', + mimeType: 'image/png', + }) + expect(mocks.storageUpload.mock.calls.at(-1)![1]).toEqual({ + lifecycle: { expiresIn: '7d' }, + }) + + await falFiles({ apiKey: 'k', expiresIn: 0 }).upload({ + data: 'AAAA', + mimeType: 'image/png', + }) + expect(mocks.storageUpload.mock.calls.at(-1)![1]).toEqual({ + lifecycle: { expiresIn: 0 }, + }) + + await falFiles({ apiKey: 'k' }).upload({ + data: 'AAAA', + mimeType: 'image/png', + }) + expect(mocks.storageUpload.mock.calls.at(-1)![1]).toBeUndefined() + }) + + it('defines no get/delete — fal storage is upload-only', () => { + // Widen to the interface (where get/delete are optional) — the class + // deliberately doesn't declare them at all. + const files: FilesAdapter<'fal'> = falFiles({ apiKey: 'k' }) + expect(files.get).toBeUndefined() + expect(files.delete).toBeUndefined() + }) +}) diff --git a/packages/ai-gemini/src/adapters/files.ts b/packages/ai-gemini/src/adapters/files.ts index 0cb9ce3ea..39c476ac4 100644 --- a/packages/ai-gemini/src/adapters/files.ts +++ b/packages/ai-gemini/src/adapters/files.ts @@ -15,7 +15,7 @@ export interface GeminiFilesConfig extends GeminiClientConfig {} * returned handle via `fileSourceFromHandle(handle)`, which uses the handle URI * (Gemini fetches it server-side as `fileData.fileUri`). */ -export class GeminiFilesAdapter extends BaseFilesAdapter { +export class GeminiFilesAdapter extends BaseFilesAdapter<'gemini'> { readonly name = 'gemini' as const private readonly client: GoogleGenAI @@ -24,7 +24,7 @@ export class GeminiFilesAdapter extends BaseFilesAdapter { this.client = createGeminiClient(config) } - async upload(input: FileUploadInput): Promise { + async upload(input: FileUploadInput): Promise> { const { blob, mimeType } = normalizeFileUploadInput(input) const file = await this.client.files.upload({ file: blob, @@ -33,7 +33,7 @@ export class GeminiFilesAdapter extends BaseFilesAdapter { return toFileHandle(file) } - async get(id: string): Promise { + async get(id: string): Promise> { return toFileHandle(await this.client.files.get({ name: id })) } @@ -42,7 +42,7 @@ export class GeminiFilesAdapter extends BaseFilesAdapter { } } -function toFileHandle(file: GeminiFile): FileHandle { +function toFileHandle(file: GeminiFile): FileHandle<'gemini'> { // `name` (e.g. "files/abc-123") is the lifecycle id; `uri` is the URL Gemini // fetches when the handle is referenced in a message. if (!file.name) { diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts index 18e5157ed..d6e3bb6fc 100644 --- a/packages/ai-gemini/src/adapters/video.ts +++ b/packages/ai-gemini/src/adapters/video.ts @@ -3,6 +3,7 @@ import { VideoGenerationReferenceType, } from '@google/genai' import { + assertOwnFileSource, isFileSource, resolveMediaPrompt, unsupportedFileSourceError, @@ -155,6 +156,11 @@ async function imagePartToVeoImage( function mediaPartToInteractionsContent( part: ImagePart | VideoPart, ): InteractionContent { + // A file handle from another provider is a bug; a Gemini handle maps to the + // `uri` field, same as a public URL (mirrors the Interactions text adapter). + if (isFileSource(part.source)) { + assertOwnFileSource(part.source, 'gemini') + } const mimeType = part.source.mimeType if (part.type === 'image') { return part.source.type === 'data' diff --git a/packages/ai-gemini/tests/files-adapter.test.ts b/packages/ai-gemini/tests/files-adapter.test.ts new file mode 100644 index 000000000..f675ee921 --- /dev/null +++ b/packages/ai-gemini/tests/files-adapter.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it, vi } from 'vitest' +import { createGeminiFiles } from '../src/adapters/files' + +const mocks = vi.hoisted(() => { + return { + filesUpload: vi.fn(), + filesGet: vi.fn(), + filesDelete: vi.fn(), + } +}) + +vi.mock('@google/genai', () => { + class MockGoogleGenAI { + files = { + upload: mocks.filesUpload, + get: mocks.filesGet, + delete: mocks.filesDelete, + } + + constructor(_options: { apiKey: string }) {} + } + return { GoogleGenAI: MockGoogleGenAI } +}) + +const GEMINI_FILE = { + name: 'files/abc-123', + uri: 'https://generativelanguage.googleapis.com/v1beta/files/abc-123', + mimeType: 'image/png', + // The SDK types sizeBytes as a string. + sizeBytes: '2048', + expirationTime: '2026-08-09T00:00:00Z', +} + +describe('gemini files adapter', () => { + it('normalizes upload results: lifecycle name as id, uri kept, ISO expiry → epoch ms, string size → number', async () => { + mocks.filesUpload.mockResolvedValueOnce(GEMINI_FILE) + const files = createGeminiFiles('k') + + const handle = await files.upload({ data: 'AAAA', mimeType: 'image/png' }) + + expect(handle).toEqual({ + id: 'files/abc-123', + provider: 'gemini', + uri: 'https://generativelanguage.googleapis.com/v1beta/files/abc-123', + mimeType: 'image/png', + sizeBytes: 2048, + expiresAt: Date.parse('2026-08-09T00:00:00Z'), + }) + }) + + it('throws when the upload response has no name — the handle would be unusable', async () => { + mocks.filesUpload.mockResolvedValueOnce({ ...GEMINI_FILE, name: undefined }) + const files = createGeminiFiles('k') + + await expect( + files.upload({ data: 'AAAA', mimeType: 'image/png' }), + ).rejects.toThrow(/without a name/) + }) + + it('omits uri and expiresAt when the API reports none', async () => { + mocks.filesUpload.mockResolvedValueOnce({ + name: 'files/abc-123', + uri: undefined, + expirationTime: undefined, + }) + const files = createGeminiFiles('k') + + const handle = await files.upload({ data: 'AAAA', mimeType: 'image/png' }) + expect(handle.uri).toBeUndefined() + expect(handle.expiresAt).toBeUndefined() + }) + + it('get/delete address the lifecycle name', async () => { + mocks.filesGet.mockResolvedValueOnce(GEMINI_FILE) + const files = createGeminiFiles('k') + + await files.get('files/abc-123') + expect(mocks.filesGet).toHaveBeenCalledWith({ name: 'files/abc-123' }) + + await files.delete('files/abc-123') + expect(mocks.filesDelete).toHaveBeenCalledWith({ name: 'files/abc-123' }) + }) +}) diff --git a/packages/ai-gemini/tests/files-source.test.ts b/packages/ai-gemini/tests/files-source.test.ts new file mode 100644 index 000000000..2f8bc927b --- /dev/null +++ b/packages/ai-gemini/tests/files-source.test.ts @@ -0,0 +1,124 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { GeminiTextAdapter } from '../src/adapters/text' +import type { StreamChunk } from '@tanstack/ai' + +const mocks = vi.hoisted(() => { + return { + generateContentStreamSpy: vi.fn(), + } +}) + +vi.mock('@google/genai', async () => { + const actual = await vi.importActual('@google/genai') + class MockGoogleGenAI { + public models = { + generateContentStream: mocks.generateContentStreamSpy, + } + + constructor(_options: { apiKey: string }) {} + } + + return { + GoogleGenAI: MockGoogleGenAI, + Type: actual.Type, + FinishReason: actual.FinishReason, + } +}) + +const emptyStream = () => + (async function* () { + yield { + candidates: [ + { content: { parts: [{ text: 'ok' }] }, finishReason: 'STOP' }, + ], + usageMetadata: { + promptTokenCount: 1, + candidatesTokenCount: 1, + totalTokenCount: 2, + }, + } + })() + +async function collectChunks( + iterable: AsyncIterable, +): Promise> { + const chunks: Array = [] + for await (const chunk of iterable) { + chunks.push(chunk) + } + return chunks +} + +describe('gemini file content source', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('maps a gemini file handle to fileData.fileUri', async () => { + mocks.generateContentStreamSpy.mockResolvedValueOnce(emptyStream()) + const adapter = new GeminiTextAdapter({ apiKey: 'k' }, 'gemini-2.5-pro') + + await collectChunks( + chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'describe' }, + { + type: 'image', + source: { + type: 'file', + value: + 'https://generativelanguage.googleapis.com/v1beta/files/abc-123', + provider: 'gemini', + mimeType: 'image/png', + }, + }, + ], + }, + ], + }), + ) + + expect(mocks.generateContentStreamSpy).toHaveBeenCalledTimes(1) + const [payload] = mocks.generateContentStreamSpy.mock.calls[0]! + const parts = payload.contents.at(-1).parts + const filePart = parts.find((p: any) => p.fileData) + expect(filePart.fileData).toEqual({ + fileUri: 'https://generativelanguage.googleapis.com/v1beta/files/abc-123', + mimeType: 'image/png', + }) + }) + + it('rejects a foreign provider file handle before any request is sent', async () => { + mocks.generateContentStreamSpy.mockResolvedValueOnce(emptyStream()) + const adapter = new GeminiTextAdapter({ apiKey: 'k' }, 'gemini-2.5-pro') + + await expect( + collectChunks( + chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'file', + value: 'file-openai-1', + provider: 'openai', + }, + }, + ], + }, + ], + }), + ), + ).rejects.toThrow(/gemini/) + expect(mocks.generateContentStreamSpy).not.toHaveBeenCalled() + }) +}) diff --git a/packages/ai-grok/tests/files-source.test.ts b/packages/ai-grok/tests/files-source.test.ts new file mode 100644 index 000000000..60fa003ef --- /dev/null +++ b/packages/ai-grok/tests/files-source.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it, vi } from 'vitest' +import { chat } from '@tanstack/ai' +import { createGrokText } from '../src/adapters/text' +import { GROK_CHAT_MODELS } from '../src/model-meta' +import type { StreamChunk } from '@tanstack/ai' + +vi.mock('openai', () => { + return { + default: class { + responses = { + create: vi.fn(), + } + }, + } +}) + +// Grok's text adapter inherits the openai-base Responses mapping, which knows +// how to emit `file_id` — but xAI has no Files API, so the base's +// `supportsFileIdInput` gate must reject file sources here instead of +// forwarding a file_id xAI can't resolve. The mocked `create` exists only to +// fail loudly if a file source ever leaks into a network call. +function adapterWithMockClient() { + const adapter = createGrokText(GROK_CHAT_MODELS[0], 'test-key') + ;(adapter as any).client = { responses: { create: vi.fn() } } + return adapter +} + +async function collectChunks( + iterable: AsyncIterable, +): Promise> { + const chunks: Array = [] + for await (const chunk of iterable) { + chunks.push(chunk) + } + return chunks +} + +function fileSourceMessage(provider: string) { + return [ + { + role: 'user' as const, + content: [ + { + type: 'image' as const, + source: { type: 'file' as const, value: 'file-abc', provider }, + }, + ], + }, + ] +} + +describe('grok file content source', () => { + it('rejects an openai file handle instead of forwarding its file_id', async () => { + const chunks = await collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: fileSourceMessage('openai'), + }), + ) + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/grok/) + expect(runError.message).toMatch(/file/) + } + }) + + it('rejects even a grok-marked file source — xAI has no Files API', async () => { + const chunks = await collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: fileSourceMessage('grok'), + }), + ) + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/does not support provider file-handle/) + } + }) +}) diff --git a/packages/ai-ollama/src/adapters/text.ts b/packages/ai-ollama/src/adapters/text.ts index 75c920ef7..47232ec4a 100644 --- a/packages/ai-ollama/src/adapters/text.ts +++ b/packages/ai-ollama/src/adapters/text.ts @@ -475,7 +475,7 @@ export class OllamaTextAdapter extends BaseTextAdapter< textContent += part.content } else if (part.type === 'image') { if (isFileSource(part.source)) { - throw unsupportedFileSourceError('ollama') + throw unsupportedFileSourceError(this.name) } images.push(part.source.value) } diff --git a/packages/ai-openai/src/adapters/files.ts b/packages/ai-openai/src/adapters/files.ts index 81e76029e..53ee1e0d3 100644 --- a/packages/ai-openai/src/adapters/files.ts +++ b/packages/ai-openai/src/adapters/files.ts @@ -23,7 +23,7 @@ export interface OpenAIFilesConfig extends OpenAIClientConfig { * it by `file_id`. Pair with `openaiText()` (Responses API): reference the * returned handle in a message via `fileSourceFromHandle(handle)`. */ -export class OpenAIFilesAdapter extends BaseFilesAdapter { +export class OpenAIFilesAdapter extends BaseFilesAdapter<'openai'> { readonly name = 'openai' as const protected client: OpenAI private readonly purpose: FilePurpose @@ -35,7 +35,7 @@ export class OpenAIFilesAdapter extends BaseFilesAdapter { this.purpose = purpose ?? 'user_data' } - async upload(input: FileUploadInput): Promise { + async upload(input: FileUploadInput): Promise> { const { blob, mimeType, filename } = normalizeFileUploadInput(input) const file = await toFile(blob, filename, { ...(mimeType ? { type: mimeType } : {}), @@ -47,7 +47,7 @@ export class OpenAIFilesAdapter extends BaseFilesAdapter { return toFileHandle(result) } - async get(id: string): Promise { + async get(id: string): Promise> { return toFileHandle(await this.client.files.retrieve(id)) } @@ -56,7 +56,7 @@ export class OpenAIFilesAdapter extends BaseFilesAdapter { } } -function toFileHandle(file: FileObject): FileHandle { +function toFileHandle(file: FileObject): FileHandle<'openai'> { return { id: file.id, provider: 'openai', diff --git a/packages/ai-openai/src/adapters/text.ts b/packages/ai-openai/src/adapters/text.ts index 2e646d1df..6657b9c07 100644 --- a/packages/ai-openai/src/adapters/text.ts +++ b/packages/ai-openai/src/adapters/text.ts @@ -92,6 +92,10 @@ export class OpenAITextAdapter< > { override readonly kind = 'text' as const override readonly name = 'openai' as const + // OpenAI's Responses endpoint consumes `file_id` references issued by its + // Files API (`openaiFiles()`); the openai-base default is false because + // compatible subclasses (Grok, Bedrock, custom) have no such surface. + protected override readonly supportsFileIdInput = true constructor(config: OpenAITextConfig, model: TModel) { super(model, 'openai', new OpenAI(config)) diff --git a/packages/ai-openai/tests/files-adapter.test.ts b/packages/ai-openai/tests/files-adapter.test.ts new file mode 100644 index 000000000..3305d00c8 --- /dev/null +++ b/packages/ai-openai/tests/files-adapter.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it, vi } from 'vitest' +import { createOpenaiFiles } from '../src/adapters/files' + +const mocks = vi.hoisted(() => { + return { + filesCreate: vi.fn(), + filesRetrieve: vi.fn(), + filesDelete: vi.fn(), + } +}) + +vi.mock('openai', () => { + class MockOpenAI { + files = { + create: mocks.filesCreate, + retrieve: mocks.filesRetrieve, + delete: mocks.filesDelete, + } + + constructor(_options: { apiKey: string }) {} + } + return { + OpenAI: MockOpenAI, + default: MockOpenAI, + toFile: vi.fn(async (blob: Blob, name?: string) => ({ blob, name })), + } +}) + +const FILE_OBJECT = { + id: 'file-abc', + bytes: 1234, + filename: 'doc.pdf', + // OpenAI reports expiry in epoch *seconds*. + expires_at: 1_700_000_000, +} + +describe('openai files adapter', () => { + it('normalizes upload results: seconds→ms expiry, provider literal, no uri', async () => { + mocks.filesCreate.mockResolvedValueOnce(FILE_OBJECT) + const files = createOpenaiFiles('k') + + const handle = await files.upload({ data: 'AAAA', mimeType: 'image/png' }) + + expect(handle).toEqual({ + id: 'file-abc', + provider: 'openai', + sizeBytes: 1234, + filename: 'doc.pdf', + expiresAt: 1_700_000_000_000, + }) + expect(handle.uri).toBeUndefined() + const [params] = mocks.filesCreate.mock.calls[0]! + expect(params.purpose).toBe('user_data') + }) + + it('omits expiresAt when the API reports none', async () => { + mocks.filesCreate.mockResolvedValueOnce({ + ...FILE_OBJECT, + expires_at: null, + }) + const files = createOpenaiFiles('k') + + const handle = await files.upload({ data: 'AAAA', mimeType: 'image/png' }) + expect(handle.expiresAt).toBeUndefined() + }) + + it('get retrieves by id and normalizes the same way', async () => { + mocks.filesRetrieve.mockResolvedValueOnce(FILE_OBJECT) + const files = createOpenaiFiles('k') + + const handle = await files.get('file-abc') + expect(mocks.filesRetrieve).toHaveBeenCalledWith('file-abc') + expect(handle.expiresAt).toBe(1_700_000_000_000) + }) +}) diff --git a/packages/ai-openai/tests/files-source.test.ts b/packages/ai-openai/tests/files-source.test.ts index 88c923846..d086c3c71 100644 --- a/packages/ai-openai/tests/files-source.test.ts +++ b/packages/ai-openai/tests/files-source.test.ts @@ -77,6 +77,76 @@ describe('openai file content source', () => { expect(imageContent.image_url).toBeUndefined() }) + it('maps a document file handle to input_file.file_id — the only supported document form', async () => { + const create = vi.fn().mockResolvedValueOnce(mockResponsesStream()) + const adapter = withMockClient(create) + + await drain( + chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'summarize' }, + { + type: 'document', + source: { + type: 'file', + value: 'file-openai-pdf', + provider: 'openai', + }, + }, + ], + }, + ], + }), + ) + + const [payload] = create.mock.calls[0]! + const userItem = payload.input.find( + (item: any) => item.type === 'message' && item.role === 'user', + ) + const fileContent = userItem.content.find( + (c: any) => c.type === 'input_file', + ) + expect(fileContent.file_id).toBe('file-openai-pdf') + }) + + it('still rejects inline (data) document parts — only handles map on this path', async () => { + const create = vi.fn().mockResolvedValueOnce(mockResponsesStream()) + const adapter = withMockClient(create) + + const chunks: Array = [] + for await (const chunk of chat({ + adapter, + messages: [ + { + role: 'user', + content: [ + { + type: 'document', + source: { + type: 'data', + value: 'AAAA', + mimeType: 'application/pdf', + }, + }, + ], + }, + ], + })) { + chunks.push(chunk) + } + + const runError = chunks.find((c) => c.type === 'RUN_ERROR') + expect(runError).toBeDefined() + if (runError?.type === 'RUN_ERROR') { + expect(runError.message).toMatch(/Unsupported content part type/) + } + expect(create).not.toHaveBeenCalled() + }) + it('errors when a foreign provider file handle reaches the openai adapter', async () => { const create = vi.fn().mockResolvedValueOnce(mockResponsesStream()) const adapter = withMockClient(create) diff --git a/packages/ai/skills/ai-core/adapter-configuration/SKILL.md b/packages/ai/skills/ai-core/adapter-configuration/SKILL.md index 454bba498..76c90919e 100644 --- a/packages/ai/skills/ai-core/adapter-configuration/SKILL.md +++ b/packages/ai/skills/ai-core/adapter-configuration/SKILL.md @@ -417,6 +417,57 @@ compatible providers speak. > Verify the provider's current `baseURL` and model ids against its live docs — > they drift. See `docs/adapters/openai-compatible.md` for the full provider table. +### 7. Files Adapters (upload once, reference by handle) + +Four providers expose a native Files/storage API as a tree-shakeable `files` +adapter: `openaiFiles()`, `anthropicFiles()`, `geminiFiles()` (each reads the +same env var as the provider's text adapter; `create*Files(apiKey)` variants +take an explicit key), and `falFiles(config)`. Upload media once with +`uploadFile()`, then reference the returned `FileHandle` in messages via a +`{ type: 'file' }` content source instead of re-sending base64 each request: + +```typescript +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { openaiFiles, openaiText } from '@tanstack/ai-openai' +import { pdfBase64 } from './pdf-data' + +const handle = await uploadFile({ + adapter: openaiFiles(), + input: { data: pdfBase64, mimeType: 'application/pdf' }, +}) + +chat({ + adapter: openaiText('gpt-5.5'), + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { type: 'document', source: fileSourceFromHandle(handle) }, + ], + }, + ], +}) +``` + +Rules agents must respect: + +- **A handle only works with the provider that issued it.** Adapters validate + `source.provider` at request time and throw on a mismatch; the `FileHandle` + provider-literal types also reject cross-provider `getFile()`/`deleteFile()` + calls at compile time. +- **Lifecycle:** `getFile()` / `deleteFile()` work for OpenAI, Anthropic, and + Gemini. fal storage is upload-only — those calls throw for `falFiles()`. +- **Not every endpoint consumes handles.** Chat Completions image inputs, + OpenAI `images/edits` + Sora `input_reference`, Gemini Veo, and providers + without a Files API (Grok, Groq, Bedrock, Mistral, OpenRouter, Ollama, + BytePlus) throw a clear "unsupported file source" error — pass `data`/`url` + sources there instead. +- `fileSourceFromHandle` and the `FileHandle` type are also exported from the + browser-safe `@tanstack/ai/client` entry for clients that persist handles. + +See `docs/advanced/files-api.md` for the full guide. + ## Common Mistakes ### a. HIGH: Confusing legacy monolithic with tree-shakeable adapter diff --git a/packages/ai/skills/ai-core/chat-experience/SKILL.md b/packages/ai/skills/ai-core/chat-experience/SKILL.md index 3136787a6..181e561db 100644 --- a/packages/ai/skills/ai-core/chat-experience/SKILL.md +++ b/packages/ai/skills/ai-core/chat-experience/SKILL.md @@ -290,6 +290,14 @@ if (part.type === 'image') { } ``` +For media reused across turns, upload once via a provider Files adapter +(`openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, `falFiles()`) and send a +`{ type: 'file' }` source built with `fileSourceFromHandle(handle)` instead of +re-sending base64 each request. `fileSourceFromHandle` is exported from the +browser-safe `@tanstack/ai/client`; the upload itself is server-side. A handle +only works with the provider that issued it. See +`ai-core/adapter-configuration/SKILL.md` §7 and `docs/advanced/files-api.md`. + ### 4. Sending Audio Messages (Browser Recording) Use `useAudioRecorder` from `@tanstack/ai-react` (or `createAudioRecorder` in Svelte) to capture audio in the browser. The resolved `AudioRecording` includes a ready-to-use `part` that slots directly into `sendMessage`. diff --git a/packages/ai/skills/ai-core/media-generation/SKILL.md b/packages/ai/skills/ai-core/media-generation/SKILL.md index e464a5f78..4cdf747cc 100644 --- a/packages/ai/skills/ai-core/media-generation/SKILL.md +++ b/packages/ai/skills/ai-core/media-generation/SKILL.md @@ -269,6 +269,14 @@ await generateVideo({ }) ``` +Reference images / start frames that are reused (or arrive as inline base64 on +memory-constrained runtimes) can instead be uploaded once via the provider's +Files adapter and referenced with `source: fileSourceFromHandle(handle)` — +supported for Gemini image generation (`geminiFiles()`) and fal image/video +inputs (`falFiles()`). Endpoints that require raw bytes (OpenAI `images/edits`, +Sora `input_reference`, Gemini Veo) reject file sources with a clear error. +See `ai-core/adapter-configuration/SKILL.md` §7. + **URL inputs that require an upload throw by default.** Most adapters pass a `type: 'url'` source straight through to the provider. Three paths can't — OpenAI `images.edit()`, OpenAI Sora `input_reference`, and Gemini **Veo** — diff --git a/packages/ai/src/activities/files/adapter.ts b/packages/ai/src/activities/files/adapter.ts index 5d62ad0a4..caef9c718 100644 --- a/packages/ai/src/activities/files/adapter.ts +++ b/packages/ai/src/activities/files/adapter.ts @@ -31,17 +31,23 @@ export type FileUploadInput = /** * A provider-issued file handle returned by {@link FilesAdapter.upload} / * {@link FilesAdapter.get}. Reference it in a message via a `{ type: 'file' }` - * content source — use {@link fileSourceFromHandle} to build one. + * content source — use `fileSourceFromHandle` to build one. + * + * `TProvider` carries the issuing provider's name as a literal (`'openai'`, + * `'gemini'`, ...) when the handle came from a concrete files adapter, so + * cross-provider lifecycle calls (`deleteFile` with a foreign handle) fail at + * compile time. It defaults to `string` so wire-deserialized handles still fit. */ -export interface FileHandle { +export interface FileHandle { /** * Provider handle used for lifecycle operations (`get`/`delete`): the * OpenAI/Anthropic `file_id`, the Gemini file resource name (`files/...`), or - * the fal storage URL. + * the fal storage URL (fal itself has no lifecycle API — the URL doubles as + * the wire reference). */ id: string /** The provider that issued the handle (`'openai'`, `'gemini'`, ...). */ - provider: string + provider: TProvider /** * The handle's URL form when the provider exposes one (Gemini file URI, fal * storage URL). For providers whose handle is an opaque id (OpenAI, @@ -61,22 +67,26 @@ export interface FileHandle { /** * The `files` adapter contract. `upload` is required; `get`/`delete` are * optional and present only when the provider has a lifecycle API. + * + * `TName` is the provider name literal (`'openai'`, `'gemini'`, ...); concrete + * adapters bind it so the handles they issue carry their provenance in the + * type system. */ -export interface FilesAdapter { +export interface FilesAdapter { readonly kind: 'files' - readonly name: string - upload: (input: FileUploadInput) => Promise - get?: (id: string) => Promise + readonly name: TName + upload: (input: FileUploadInput) => Promise> + get?: (id: string) => Promise> delete?: (id: string) => Promise } -export type AnyFilesAdapter = FilesAdapter +export type AnyFilesAdapter = FilesAdapter /** * Normalize a {@link FileUploadInput} to a `Blob` (plus best-effort MIME / * filename) so provider adapters can hand it straight to their SDK. A `Blob` - * input passes through; base64 `{ data }` is decoded to bytes. Shared so the - * four provider files adapters don't each re-implement the decode. + * input passes through; base64 `{ data }` is decoded to bytes. Shared so + * provider files adapters don't each re-implement the decode. */ export function normalizeFileUploadInput(input: FileUploadInput): { blob: Blob @@ -95,12 +105,16 @@ export function normalizeFileUploadInput(input: FileUploadInput): { } /** - * Abstract base for provider files adapters. Subclasses set `name`, implement - * `upload`, and optionally implement `get`/`delete`. + * Abstract base for provider files adapters. Subclasses bind `TName` to their + * provider literal, set `name`, implement `upload`, and may add `get`/`delete` + * (declared on {@link FilesAdapter}, not here, since not every provider has a + * lifecycle API). */ -export abstract class BaseFilesAdapter implements FilesAdapter { +export abstract class BaseFilesAdapter< + TName extends string = string, +> implements FilesAdapter { readonly kind = 'files' as const - abstract readonly name: string + abstract readonly name: TName - abstract upload(input: FileUploadInput): Promise + abstract upload(input: FileUploadInput): Promise> } diff --git a/packages/ai/src/activities/files/index.ts b/packages/ai/src/activities/files/index.ts index 5a098fa78..dc6b800a5 100644 --- a/packages/ai/src/activities/files/index.ts +++ b/packages/ai/src/activities/files/index.ts @@ -8,13 +8,15 @@ */ import type { ContentPartFileSource } from '../../types' -import type { AnyFilesAdapter, FileHandle, FileUploadInput } from './adapter' +import type { FileHandle, FileUploadInput, FilesAdapter } from './adapter' /** The adapter kind this activity handles */ export const kind = 'files' as const /** - * Upload a file to a provider's Files API and return its handle. + * Upload a file to a provider's Files API and return its handle. The handle + * carries the provider name as a literal type, so passing it to another + * provider's lifecycle call is a compile error. * * @example * ```ts @@ -22,49 +24,61 @@ export const kind = 'files' as const * const handle = await uploadFile({ adapter: files, input: { data, mimeType: 'image/png' } }) * ``` */ -export async function uploadFile(options: { - adapter: TAdapter & { kind: typeof kind } +export async function uploadFile(options: { + adapter: FilesAdapter & { kind: typeof kind } input: FileUploadInput -}): Promise { +}): Promise> { return options.adapter.upload(options.input) } /** - * Fetch metadata for a previously uploaded file by its handle id. + * Resolve a lifecycle id from either a raw id string or a {@link FileHandle} + * (whose `id` — not its `uri`/wire value — is the lifecycle currency). + */ +function toLifecycleId(id: string | FileHandle): string { + return typeof id === 'string' ? id : id.id +} + +/** + * Fetch metadata for a previously uploaded file. Accepts the handle itself + * (preferred — the provider-literal type rejects a foreign provider's handle + * at compile time) or its raw lifecycle id. * * @throws if the provider's files adapter has no `get` (e.g. fal storage). */ -export async function getFile(options: { - adapter: TAdapter & { kind: typeof kind } - id: string -}): Promise { - const { adapter, id } = options +export async function getFile(options: { + adapter: FilesAdapter & { kind: typeof kind } + id: string | FileHandle +}): Promise> { + const { adapter } = options if (!adapter.get) { throw new Error( `${adapter.name}: files adapter does not support get() — this provider ` + `has no file-retrieval API.`, ) } - return adapter.get(id) + return adapter.get(toLifecycleId(options.id)) } /** - * Delete a previously uploaded file by its handle id. + * Delete a previously uploaded file. Accepts the handle itself (preferred — + * the provider-literal type rejects a foreign provider's handle at compile + * time) or its raw lifecycle id. * * @throws if the provider's files adapter has no `delete` (e.g. fal storage). */ -export async function deleteFile(options: { - adapter: TAdapter & { kind: typeof kind } - id: string +export async function deleteFile(options: { + adapter: FilesAdapter & { kind: typeof kind } + id: string | FileHandle }): Promise { - const { adapter, id } = options + const { adapter } = options if (!adapter.delete) { throw new Error( `${adapter.name}: files adapter does not support delete() — this ` + `provider has no file-deletion API.`, ) } - return adapter.delete(id) + return adapter.delete(toLifecycleId(options.id)) } /** @@ -82,9 +96,9 @@ export async function deleteFile(options: { * ] }) * ``` */ -export function fileSourceFromHandle( - handle: FileHandle, -): ContentPartFileSource { +export function fileSourceFromHandle( + handle: FileHandle, +): ContentPartFileSource { return { type: 'file', value: handle.uri ?? handle.id, diff --git a/packages/ai/src/client.ts b/packages/ai/src/client.ts index 4656708bb..98f2b3326 100644 --- a/packages/ai/src/client.ts +++ b/packages/ai/src/client.ts @@ -299,6 +299,13 @@ export type { export { uiMessagesToWire } from './utilities/ag-ui-wire' export type { WireMessage } from './utilities/ag-ui-wire' +// A browser client that received an uploaded handle from its server can build +// the `{ type: 'file' }` content source itself — `fileSourceFromHandle` is a +// pure object builder, so exporting it here keeps the documented client flow +// from pulling in the server entry. +export { fileSourceFromHandle } from './activities/files/index' +export type { FileHandle } from './activities/files/adapter' + export type { AudioPart, ContentPart, diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 3979e351d..22efea533 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -254,10 +254,10 @@ export interface ContentPartUrlSource { * A handle only routes to the provider that issued it — the `provider` field is * validated at map time, and adapters throw if it doesn't match. The `value` is * the provider's opaque id (OpenAI/Anthropic `file_id`) or handle URL (Gemini - * file URI, fal storage URL); use {@link fileSourceFromHandle} to build one from - * a {@link FileHandle} without worrying about the id-vs-uri distinction. + * file URI, fal storage URL); use `fileSourceFromHandle` to build one from a + * `FileHandle` without worrying about the id-vs-uri distinction. */ -export interface ContentPartFileSource { +export interface ContentPartFileSource { /** * Indicates this references a provider-issued file handle. */ @@ -271,7 +271,7 @@ export interface ContentPartFileSource { * The provider that issued the handle. A handle is only valid for its issuer; * passing (e.g.) an OpenAI `file-...` id to a Gemini adapter is an error. */ - provider: string + provider: TProvider /** * Optional MIME type hint for cases where the provider can't infer it. */ diff --git a/packages/ai/src/utilities/content-source.ts b/packages/ai/src/utilities/content-source.ts index f61ad3432..e7bac5615 100644 --- a/packages/ai/src/utilities/content-source.ts +++ b/packages/ai/src/utilities/content-source.ts @@ -43,7 +43,9 @@ export function assertOwnFileSource( * (image edits, Veo) rather than a reference. * * @param detail Optional context appended to the message (e.g. a modality or - * endpoint name, or a pointer to the adapter that does support handles). + * endpoint name, or a pointer to the adapter that does support handles). When + * provided it replaces the generic remediation tail, so a site-specific hint + * ("pass inline bytes") is never contradicted by generic advice. */ export function unsupportedFileSourceError( providerName: string, @@ -51,7 +53,10 @@ export function unsupportedFileSourceError( ): Error { return new Error( `${providerName} does not support provider file-handle sources ` + - `({ type: 'file' })${detail ? ` ${detail}` : ''}. Pass a data or url ` + - `source, or upload via the provider's files adapter where supported.`, + `({ type: 'file' })` + + (detail + ? ` ${detail}.` + : `. Pass a data or url source, or upload via the provider's files ` + + `adapter where supported.`), ) } diff --git a/packages/ai/tests/files-source.test.ts b/packages/ai/tests/files-source.test.ts index d8e41cd28..691dcc8aa 100644 --- a/packages/ai/tests/files-source.test.ts +++ b/packages/ai/tests/files-source.test.ts @@ -128,4 +128,30 @@ describe('files activity dispatch', () => { deleteFile({ adapter: full, id: 'file-1' }), ).resolves.toBeUndefined() }) + + it('getFile / deleteFile accept the handle itself and use its lifecycle id', async () => { + // A Gemini-style handle: `uri` is the wire value, `id` the lifecycle name. + const handle: FileHandle = { + id: 'file-1', + provider: 'openai', + uri: 'https://provider/file-1', + } + const seen: Array = [] + const adapter: FilesAdapter = { + kind: 'files', + name: 'openai', + upload: async () => handle, + get: async (id) => { + seen.push(id) + return handle + }, + delete: async (id) => { + seen.push(id) + }, + } + + await getFile({ adapter, id: handle }) + await deleteFile({ adapter, id: handle }) + expect(seen).toEqual(['file-1', 'file-1']) + }) }) diff --git a/packages/openai-base/src/adapters/chat-completions-text.ts b/packages/openai-base/src/adapters/chat-completions-text.ts index 68751777a..1742566cb 100644 --- a/packages/openai-base/src/adapters/chat-completions-text.ts +++ b/packages/openai-base/src/adapters/chat-completions-text.ts @@ -1318,11 +1318,18 @@ export abstract class OpenAIBaseChatCompletionsTextAdapter< | undefined if (isFileSource(part.source)) { - // The Chat Completions API references images only by URL/data URI, not - // by an uploaded file_id — point callers at the Responses adapter. + // The Chat Completions API references images only by URL/data URI, + // not by an uploaded file_id. Only mention the Responses alternative + // for OpenAI itself — compatible providers built on this base have no + // file_id-consuming endpoint at all. throw unsupportedFileSourceError( this.name, - 'on the Chat Completions API — use the Responses adapter (e.g. openaiText) to reference an uploaded file by file_id', + // Only OpenAI's own chat-completions adapter (name 'openai-chat') + // has a Responses sibling to point at; other subclasses (Groq, + // Bedrock, BytePlus, compatible providers) have no file_id surface. + this.name === 'openai-chat' + ? 'on the Chat Completions API — use the Responses adapter (openaiText) to reference an uploaded file by file_id' + : 'on the Chat Completions API', ) } diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index dc5b6fe02..a03ff9205 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -3,6 +3,7 @@ import { assertOwnFileSource, isFileSource, normalizeSystemPrompts, + unsupportedFileSourceError, } from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { @@ -60,6 +61,15 @@ export abstract class OpenAIBaseResponsesTextAdapter< readonly name: string protected client: OpenAI + /** + * Whether this provider's Responses endpoint accepts `file_id` references + * from its own Files API. Only OpenAI itself does — OpenAI-compatible + * subclasses (Grok, Bedrock, custom providers) have no Files API surface, + * so a `{ type: 'file' }` source is rejected instead of being sent as a + * `file_id` the provider can't resolve. + */ + protected readonly supportsFileIdInput: boolean = false + constructor(model: TModel, name: string, client: OpenAI) { super({}, model) this.name = name @@ -1805,6 +1815,9 @@ export abstract class OpenAIBaseResponsesTextAdapter< | { detail?: 'auto' | 'low' | 'high' } | undefined if (isFileSource(part.source)) { + if (!this.supportsFileIdInput) { + throw unsupportedFileSourceError(this.name) + } assertOwnFileSource(part.source, this.name) return { type: 'input_image', @@ -1836,6 +1849,9 @@ export abstract class OpenAIBaseResponsesTextAdapter< } case 'audio': { if (isFileSource(part.source)) { + if (!this.supportsFileIdInput) { + throw unsupportedFileSourceError(this.name) + } assertOwnFileSource(part.source, this.name) return { type: 'input_file', @@ -1863,9 +1879,13 @@ export abstract class OpenAIBaseResponsesTextAdapter< } case 'document': { - // A document uploaded via the Files API is referenced by `file_id`; - // inline document bytes/URLs aren't accepted on this path. + // A document uploaded via the Files API is referenced by `file_id`. + // This adapter doesn't map inline document bytes/URLs onto the + // Responses `file_data`/`file_url` fields (yet) — only handles. if (isFileSource(part.source)) { + if (!this.supportsFileIdInput) { + throw unsupportedFileSourceError(this.name) + } assertOwnFileSource(part.source, this.name) return { type: 'input_file', @@ -1876,9 +1896,9 @@ export abstract class OpenAIBaseResponsesTextAdapter< } case 'video': default: - // OpenAI Responses API doesn't accept native video/document parts on - // this path — surface as explicit unsupported error so callers see - // the same message regardless of which content type leaked through. + // The Responses API doesn't accept native video parts on this path — + // surface as explicit unsupported error so callers see the same + // message regardless of which content type leaked through. throw new Error(`Unsupported content part type: ${part.type}`) } } diff --git a/testing/e2e/src/routeTree.gen.ts b/testing/e2e/src/routeTree.gen.ts index 68e376674..f63323387 100644 --- a/testing/e2e/src/routeTree.gen.ts +++ b/testing/e2e/src/routeTree.gen.ts @@ -59,6 +59,7 @@ import { Route as ApiImageRouteImport } from './routes/api.image' import { Route as ApiGenerationPersistenceServerRouteImport } from './routes/api.generation-persistence-server' import { Route as ApiGenerationPersistenceResumeRouteImport } from './routes/api.generation-persistence-resume' import { Route as ApiForeignInterruptRouteImport } from './routes/api.foreign-interrupt' +import { Route as ApiFileSourceWireRouteImport } from './routes/api.file-source-wire' import { Route as ApiDurableTakeoverRouteImport } from './routes/api.durable-takeover' import { Route as ApiDurableDeliveryRouteImport } from './routes/api.durable-delivery' import { Route as ApiDevtoolsMemoryRouteImport } from './routes/api.devtools-memory' @@ -334,6 +335,11 @@ const ApiForeignInterruptRoute = ApiForeignInterruptRouteImport.update({ path: '/api/foreign-interrupt', getParentRoute: () => rootRouteImport, } as any) +const ApiFileSourceWireRoute = ApiFileSourceWireRouteImport.update({ + id: '/api/file-source-wire', + path: '/api/file-source-wire', + getParentRoute: () => rootRouteImport, +} as any) const ApiDurableTakeoverRoute = ApiDurableTakeoverRouteImport.update({ id: '/api/durable-takeover', path: '/api/durable-takeover', @@ -439,6 +445,7 @@ export interface FileRoutesByFullPath { '/api/devtools-memory': typeof ApiDevtoolsMemoryRoute '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute + '/api/file-source-wire': typeof ApiFileSourceWireRoute '/api/foreign-interrupt': typeof ApiForeignInterruptRoute '/api/generation-persistence-resume': typeof ApiGenerationPersistenceResumeRoute '/api/generation-persistence-server': typeof ApiGenerationPersistenceServerRoute @@ -506,6 +513,7 @@ export interface FileRoutesByTo { '/api/devtools-memory': typeof ApiDevtoolsMemoryRoute '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute + '/api/file-source-wire': typeof ApiFileSourceWireRoute '/api/foreign-interrupt': typeof ApiForeignInterruptRoute '/api/generation-persistence-resume': typeof ApiGenerationPersistenceResumeRoute '/api/generation-persistence-server': typeof ApiGenerationPersistenceServerRoute @@ -574,6 +582,7 @@ export interface FileRoutesById { '/api/devtools-memory': typeof ApiDevtoolsMemoryRoute '/api/durable-delivery': typeof ApiDurableDeliveryRoute '/api/durable-takeover': typeof ApiDurableTakeoverRoute + '/api/file-source-wire': typeof ApiFileSourceWireRoute '/api/foreign-interrupt': typeof ApiForeignInterruptRoute '/api/generation-persistence-resume': typeof ApiGenerationPersistenceResumeRoute '/api/generation-persistence-server': typeof ApiGenerationPersistenceServerRoute @@ -643,6 +652,7 @@ export interface FileRouteTypes { | '/api/devtools-memory' | '/api/durable-delivery' | '/api/durable-takeover' + | '/api/file-source-wire' | '/api/foreign-interrupt' | '/api/generation-persistence-resume' | '/api/generation-persistence-server' @@ -710,6 +720,7 @@ export interface FileRouteTypes { | '/api/devtools-memory' | '/api/durable-delivery' | '/api/durable-takeover' + | '/api/file-source-wire' | '/api/foreign-interrupt' | '/api/generation-persistence-resume' | '/api/generation-persistence-server' @@ -777,6 +788,7 @@ export interface FileRouteTypes { | '/api/devtools-memory' | '/api/durable-delivery' | '/api/durable-takeover' + | '/api/file-source-wire' | '/api/foreign-interrupt' | '/api/generation-persistence-resume' | '/api/generation-persistence-server' @@ -845,6 +857,7 @@ export interface RootRouteChildren { ApiDevtoolsMemoryRoute: typeof ApiDevtoolsMemoryRoute ApiDurableDeliveryRoute: typeof ApiDurableDeliveryRoute ApiDurableTakeoverRoute: typeof ApiDurableTakeoverRoute + ApiFileSourceWireRoute: typeof ApiFileSourceWireRoute ApiForeignInterruptRoute: typeof ApiForeignInterruptRoute ApiGenerationPersistenceResumeRoute: typeof ApiGenerationPersistenceResumeRoute ApiGenerationPersistenceServerRoute: typeof ApiGenerationPersistenceServerRoute @@ -1232,6 +1245,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof ApiForeignInterruptRouteImport parentRoute: typeof rootRouteImport } + '/api/file-source-wire': { + id: '/api/file-source-wire' + path: '/api/file-source-wire' + fullPath: '/api/file-source-wire' + preLoaderRoute: typeof ApiFileSourceWireRouteImport + parentRoute: typeof rootRouteImport + } '/api/durable-takeover': { id: '/api/durable-takeover' path: '/api/durable-takeover' @@ -1426,6 +1446,7 @@ const rootRouteChildren: RootRouteChildren = { ApiDevtoolsMemoryRoute: ApiDevtoolsMemoryRoute, ApiDurableDeliveryRoute: ApiDurableDeliveryRoute, ApiDurableTakeoverRoute: ApiDurableTakeoverRoute, + ApiFileSourceWireRoute: ApiFileSourceWireRoute, ApiForeignInterruptRoute: ApiForeignInterruptRoute, ApiGenerationPersistenceResumeRoute: ApiGenerationPersistenceResumeRoute, ApiGenerationPersistenceServerRoute: ApiGenerationPersistenceServerRoute, diff --git a/testing/e2e/src/routes/api.file-source-wire.ts b/testing/e2e/src/routes/api.file-source-wire.ts new file mode 100644 index 000000000..01f3e794a --- /dev/null +++ b/testing/e2e/src/routes/api.file-source-wire.ts @@ -0,0 +1,98 @@ +import { createFileRoute } from '@tanstack/react-router' +import { chat, createChatOptions } from '@tanstack/ai' +import { createTextAdapter } from '@/lib/providers' +import type { ModelMessage } from '@tanstack/ai' +import type { Provider } from '@/lib/types' + +/** + * Wire-format verification for `{ type: 'file' }` content sources (#909). + * + * A message image part can reference a provider-issued Files API handle + * instead of inline base64 / a URL. This route drives a single `chat()` call + * whose user message carries a file source so the companion spec can inspect + * aimock's journal (`GET /v1/_requests`) and assert the adapter emitted the + * provider's native file reference (OpenAI `input_image.file_id`, Anthropic + * `file_id` source, Gemini `fileData.fileUri`). + * + * `handleProvider` defaults to the target provider (the supported case); pass + * a different one to drive the cross-provider rejection path. + */ +export const Route = createFileRoute('/api/file-source-wire')({ + server: { + handlers: { + POST: async ({ request }) => { + const url = new URL(request.url) + const provider = (url.searchParams.get('provider') ?? + 'openai') as Provider + const handleProvider = + url.searchParams.get('handleProvider') ?? provider + const testId = url.searchParams.get('testId') ?? undefined + + const { adapter } = createTextAdapter( + provider, + undefined, + undefined, + testId, + ) + + // A synthetic handle in each provider's shape — uploads themselves + // can't run against aimock, but the wire mapping doesn't care where + // the handle came from. + const handleValue = + handleProvider === 'gemini' + ? 'https://generativelanguage.googleapis.com/v1beta/files/e2e-abc' + : 'file-e2e-abc' + + const messages: Array = [ + { + role: 'user', + content: [ + { type: 'text', content: 'Describe the attached image.' }, + { + type: 'image', + source: { + type: 'file', + value: handleValue, + provider: handleProvider, + }, + }, + ], + }, + ] + + // Adapters differ in how the cross-provider guard surfaces: some + // yield a RUN_ERROR chunk, others throw before the stream starts. + // Report both the same way so the spec has one shape to assert. + let runError: string | undefined + try { + for await (const chunk of chat({ + ...createChatOptions({ adapter }), + messages, + })) { + if ( + chunk.type === 'RUN_ERROR' && + /file/.test(chunk.message ?? '') + ) { + runError = chunk.message + } + } + } catch (error) { + return new Response( + JSON.stringify({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ) + } + + return new Response( + JSON.stringify( + runError ? { ok: false, error: runError } : { ok: true }, + ), + { status: 200, headers: { 'Content-Type': 'application/json' } }, + ) + }, + }, + }, +}) diff --git a/testing/e2e/tests/file-source-wire.spec.ts b/testing/e2e/tests/file-source-wire.spec.ts new file mode 100644 index 000000000..04d9bc245 --- /dev/null +++ b/testing/e2e/tests/file-source-wire.spec.ts @@ -0,0 +1,97 @@ +import { test, expect } from './fixtures' + +/** + * Wire-format verification for `{ type: 'file' }` content sources (#909). + * + * Uploads can't run against aimock, so the route at `/api/file-source-wire` + * drives `chat()` with a synthetic provider handle and this spec inspects + * aimock's journal (`GET /v1/_requests`) for the provider's native file + * reference. Aimock normalises every request body to an OpenAI-compatible + * chat form before journalling, and that normalisation strips user-message + * image parts for all three providers — so the positive assertions here are + * the end-to-end ok:true round-trip (the mapping ran to completion and the + * request reached the provider endpoint), with the structural wire proof in + * the per-package unit tests (packages/ai-openai/tests/files-source.test.ts, + * packages/ai-anthropic/tests/files-source.test.ts, + * packages/ai-gemini/tests/files-source.test.ts). The cross-provider + * rejection path IS asserted end-to-end, for both a RUN_ERROR-yielding + * adapter (OpenAI) and a throwing one (Gemini). + */ +// Serial mode: each test clears then re-populates the aimock journal. +test.describe.configure({ mode: 'serial' }) + +test.describe('file content source — wire format', () => { + test.beforeEach(async ({ request, aimockPort }) => { + await request.delete(`http://127.0.0.1:${aimockPort}/v1/_requests`) + }) + + test('openai: an own-provider handle completes the round-trip (input_image.file_id covered by unit test)', async ({ + request, + aimockPort, + testId, + }) => { + const res = await request.post( + `/api/file-source-wire?provider=openai&testId=${encodeURIComponent(testId)}`, + ) + const { ok } = (await res.json()) as { ok: boolean; error?: string } + expect(ok).toBe(true) + // The request must actually have reached the mock endpoint — ok:true with + // an empty journal would mean the call never left the adapter. + const journal = await request.get( + `http://127.0.0.1:${aimockPort}/v1/_requests`, + ) + const entries = (await journal.json()) as Array<{ body: any }> + expect(entries.length).toBeGreaterThan(0) + }) + + test('openai: a foreign (gemini) handle is rejected, not forwarded', async ({ + request, + testId, + }) => { + const res = await request.post( + `/api/file-source-wire?provider=openai&handleProvider=gemini&testId=${encodeURIComponent(testId)}`, + ) + const { ok, error } = (await res.json()) as { ok: boolean; error?: string } + expect(ok).toBe(false) + expect(error).toMatch(/openai/) + expect(error).toMatch(/gemini/) + }) + + test('anthropic: an own-provider handle completes the round-trip (file_id block covered by unit test)', async ({ + request, + testId, + }) => { + const res = await request.post( + `/api/file-source-wire?provider=anthropic&testId=${encodeURIComponent(testId)}`, + ) + const { ok } = (await res.json()) as { ok: boolean; error?: string } + // Structural proof that the handle becomes a { type: 'file', file_id } + // source with the files-api beta lives in + // packages/ai-anthropic/tests/files-source.test.ts — aimock's journal + // normalisation strips the block so it can't be asserted here. + expect(ok).toBe(true) + }) + + test('gemini: an own-provider handle completes the round-trip (fileData.fileUri covered by unit test)', async ({ + request, + testId, + }) => { + const res = await request.post( + `/api/file-source-wire?provider=gemini&testId=${encodeURIComponent(testId)}`, + ) + const { ok } = (await res.json()) as { ok: boolean; error?: string } + expect(ok).toBe(true) + }) + + test('gemini: a foreign (openai) handle is rejected before any request', async ({ + request, + testId, + }) => { + const res = await request.post( + `/api/file-source-wire?provider=gemini&handleProvider=openai&testId=${encodeURIComponent(testId)}`, + ) + const { ok, error } = (await res.json()) as { ok: boolean; error?: string } + expect(ok).toBe(false) + expect(error).toMatch(/gemini/) + }) +}) From aaeb09056ea215b0b07ba7fcc713408b09fec0a0 Mon Sep 17 00:00:00 2001 From: Tom Beckenham <34339192+tombeckenham@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:06:46 +1000 Subject: [PATCH 5/5] refactor(files): record-based file references + fail-closed capability preflight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redesigns the `{ type: 'file' }` content source before it ships: - The source now carries a per-provider reference record — `{ type: 'file', reference: { openai: 'file-…', gemini: 'https://…' } }` — instead of a single { value, provider } pair. Each adapter reads only its own entry (`fileReferenceFor`), and a lookup miss throws naming the providers that are present. `fileSourceFromHandle(...handles)` merges handles from several providers into one source that routes to any of them (upload once per provider, replay the same conversation anywhere). - New fail-closed preflight: adapters that can consume file references declare `supportsFileSources`; chat()/generateImage()/generateVideo() reject file sources for every other adapter before a request is built. Adapters written before this feature existed can no longer silently mis-map a reference onto their URL/data branch — the failure class the per-adapter sweep was policing by convention is now structural. (openai-base's supportsFileIdInput gate is folded into the same flag.) - Removing `value` from the file arm also makes fall-through code a compile error — caught two latent OpenRouter paths that would have sent a reference as a URL, now restructured with narrowed sources. - Docs, agent skills, changeset, and all files-source tests updated to the record shape; new preflight unit tests; e2e spec asserts the record round-trip and lookup-miss rejection end-to-end. --- .changeset/native-files-api-support.md | 4 +- docs/advanced/files-api.md | 39 +++++- docs/advanced/multimodal-content.md | 4 +- examples/ts-react-chat/src/routes/index.tsx | 5 +- packages/ai-anthropic/src/adapters/text.ts | 16 ++- .../ai-anthropic/tests/files-source.test.ts | 6 +- .../ai-byteplus/tests/files-source.test.ts | 95 +++++++-------- packages/ai-event-client/src/index.ts | 4 +- packages/ai-fal/src/adapters/image.ts | 2 + packages/ai-fal/src/adapters/video.ts | 2 + packages/ai-fal/src/image/image-inputs.ts | 7 +- .../tests/content-source-to-fal-url.test.ts | 6 +- packages/ai-gemini/src/adapters/image.ts | 17 +-- packages/ai-gemini/src/adapters/text.ts | 27 +++-- packages/ai-gemini/src/adapters/video.ts | 24 ++-- .../experimental/text-interactions/adapter.ts | 32 ++--- packages/ai-gemini/tests/files-source.test.ts | 10 +- packages/ai-grok/tests/files-source.test.ts | 50 ++++---- packages/ai-openai/src/adapters/text.ts | 7 +- packages/ai-openai/tests/files-source.test.ts | 9 +- .../src/adapters/responses-text.ts | 49 +++++--- packages/ai-openrouter/src/adapters/text.ts | 33 +++-- .../ai-core/adapter-configuration/SKILL.md | 27 +++-- .../skills/ai-core/chat-experience/SKILL.md | 11 +- packages/ai/src/activities/chat/adapter.ts | 10 ++ packages/ai/src/activities/chat/index.ts | 8 ++ packages/ai/src/activities/files/index.ts | 30 +++-- .../src/activities/generateImage/adapter.ts | 8 ++ .../ai/src/activities/generateImage/index.ts | 5 + .../src/activities/generateVideo/adapter.ts | 8 ++ .../ai/src/activities/generateVideo/index.ts | 7 ++ packages/ai/src/index.ts | 5 +- packages/ai/src/types.ts | 28 ++--- packages/ai/src/utilities/content-source.ts | 114 ++++++++++++++---- packages/ai/src/utilities/tool-result.ts | 10 +- packages/ai/tests/files-source.test.ts | 89 ++++++++++++-- packages/ai/tests/media-prompt.test.ts | 9 +- .../src/adapters/responses-text.ts | 26 ++-- .../e2e/src/routes/api.file-source-wire.ts | 3 +- testing/e2e/tests/file-source-wire.spec.ts | 7 +- 40 files changed, 551 insertions(+), 302 deletions(-) diff --git a/.changeset/native-files-api-support.md b/.changeset/native-files-api-support.md index 3b355fdb5..8606e4e01 100644 --- a/.changeset/native-files-api-support.md +++ b/.changeset/native-files-api-support.md @@ -19,6 +19,6 @@ feat(ai): native Files API support across providers (upload adapters + `file` co Adds first-class support for provider **Files / storage APIs** so callers can upload media once and reference it by a provider-issued handle instead of re-sending base64 or a public URL each request (lower latency/bandwidth, no re-buffering on memory-constrained runtimes). - **New tree-shakeable `files` adapter kind** — `openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, and `falFiles()`. Each exposes `upload()`, and (where the provider has a lifecycle API) `get()` / `delete()`. Drive them with the new `uploadFile()` / `getFile()` / `deleteFile()` activity functions. fal is upload-only. -- **New `{ type: 'file' }` arm on `ContentPartSource`** — reference an uploaded handle in a chat message. Adapters map it to the right wire field: OpenAI (Responses) `input_image`/`input_file` `file_id`, Anthropic `file_id` message source (with the `files-api-2025-04-14` beta), Gemini `fileData.fileUri`, fal storage URL passthrough. Use `fileSourceFromHandle(handle)` to build the source from an uploaded `FileHandle`. -- **Runtime provider routing** — a file handle only routes to the provider that issued it; adapters throw a clear error on a cross-provider handle, and providers/endpoints that can't consume a handle (image edits, Veo, Chat Completions images, Bedrock, Mistral, Grok, OpenRouter, Ollama, BytePlus) throw a clear "unsupported file source" error instead of silently mis-mapping. Grok/Bedrock adapters built on the shared OpenAI Responses base are gated too (`supportsFileIdInput`), so an inherited `file_id` mapping can't leak to providers without a Files API. +- **New `{ type: 'file' }` arm on `ContentPartSource`** — a **per-provider reference record**: `{ type: 'file', reference: { openai: 'file-…', gemini: 'https://…' } }`. Each adapter reads only its own entry and maps it to its native wire field: OpenAI (Responses) `input_image`/`input_file` `file_id`, Anthropic `file_id` message source (with the `files-api-2025-04-14` beta), Gemini `fileData.fileUri`, fal storage URL passthrough. `fileSourceFromHandle(...handles)` builds the source and merges handles from several providers into one source that routes to any of them. +- **Fail-closed capability preflight** — adapters that can consume file references declare `supportsFileSources`; `chat()` / `generateImage()` / `generateVideo()` reject `{ type: 'file' }` sources for every other adapter (Bedrock, Mistral, Grok, Groq, OpenRouter, Ollama, BytePlus, and any future adapter that doesn't opt in) **before a request is built**, so a reference can never be silently mis-mapped onto a URL/data field. Endpoints that need raw bytes (image edits, Sora `input_reference`, Veo, Chat Completions images) throw endpoint-specific errors. A supporting adapter with no entry for its provider in the record throws a lookup error naming the providers that are present. - **Provider-literal typed handles** — `FileHandle<'openai'>` etc. flow from each files adapter through `uploadFile()`, and `getFile()`/`deleteFile()` accept the handle itself, so cross-provider lifecycle calls fail at compile time. `fileSourceFromHandle` and `FileHandle` are also exported from the browser-safe `@tanstack/ai/client` entry. diff --git a/docs/advanced/files-api.md b/docs/advanced/files-api.md index d23f87ca9..f2774e070 100644 --- a/docs/advanced/files-api.md +++ b/docs/advanced/files-api.md @@ -81,7 +81,7 @@ await deleteFile({ adapter: files, id: handle }) ## Referencing a handle in a message -Use `fileSourceFromHandle(handle)` to turn a `FileHandle` into a `{ type: 'file' }` content source. Each adapter maps it to the provider's native reference (OpenAI/Anthropic `file_id`, Gemini `fileData.fileUri`, fal storage URL). A handle only works with the provider that issued it — passing it elsewhere throws. +Use `fileSourceFromHandle(handle)` to turn a `FileHandle` into a `{ type: 'file' }` content source. The source carries a **record of per-provider references** — `{ reference: { openai: 'file-abc' } }` — and each adapter reads only its own entry, mapping it to its native wire field (OpenAI/Anthropic `file_id`, Gemini `fileData.fileUri`, fal storage URL). Sending the source to a provider with no entry in the record throws a clear error, and adapters that can't consume file references at all are rejected before any mapping starts. ### Server: upload + reference @@ -111,6 +111,37 @@ export async function askAboutPdf(pdfBase64: string, request: string) { } ``` +### One source, several providers + +Because `reference` is a record, the same bytes uploaded to two providers merge into **one** source that routes correctly to either — useful when a conversation may be replayed against different models: + +```typescript +import { chat, fileSourceFromHandle, uploadFile } from '@tanstack/ai' +import { openaiFiles, openaiText } from '@tanstack/ai-openai' +import { geminiFiles } from '@tanstack/ai-gemini' +import { pdfBase64 } from './pdf-data' + +const input = { data: pdfBase64, mimeType: 'application/pdf' } +const openaiHandle = await uploadFile({ adapter: openaiFiles(), input }) +const geminiHandle = await uploadFile({ adapter: geminiFiles(), input }) + +// reference: { openai: 'file-…', gemini: 'https://…/files/…' } +const source = fileSourceFromHandle(openaiHandle, geminiHandle) + +chat({ + adapter: openaiText('gpt-5.5'), // or a gemini adapter — same message works + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this document' }, + { type: 'document', source }, + ], + }, + ], +}) +``` + ### Client: reuse a handle across requests Upload happens server-side (it needs the provider key), so the client works with the returned handle. Persist `{ id, provider, uri, mimeType }` and rebuild the source on each turn. `fileSourceFromHandle` and `FileHandle` are exported from the browser-safe `@tanstack/ai/client` entry, so this doesn't pull the server bundle into the client: @@ -142,6 +173,8 @@ function imageMessage(handle: FileHandle, prompt: string) { Gemini and fal handles are URLs, so they also round-trip through a plain `{ type: 'url' }` source; OpenAI and Anthropic handles are opaque ids that require the `{ type: 'file' }` source. -### Endpoints that require raw bytes +### Providers and endpoints that can't consume references + +Adapters that can consume file references declare a `supportsFileSources` capability; for everyone else — Grok, Groq, Bedrock, Mistral, OpenRouter, Ollama, BytePlus, and any adapter written before this feature existed — `chat()` / `generateImage()` / `generateVideo()` reject `{ type: 'file' }` sources **before any request is built**, so a reference can never be silently mis-mapped onto a URL or data field. -Some endpoints have no "reference an uploaded handle" option — OpenAI's `images/edits` and Sora `input_reference`, and Gemini's Veo, need the actual bytes (or, for Veo, a `gs://` URI). The OpenAI **Chat Completions** image path also references images only by URL/data URI, not `file_id` — use the Responses adapter (`openaiText`) for `file_id` images. Passing a `{ type: 'file' }` source to any of these throws a clear error rather than silently mis-mapping. +Some endpoints on supporting providers also have no "reference an uploaded handle" option — OpenAI's `images/edits` and Sora `input_reference`, and Gemini's Veo, need the actual bytes (or, for Veo, a `gs://` URI). The OpenAI **Chat Completions** image path also references images only by URL/data URI, not `file_id` — use the Responses adapter (`openaiText`) for `file_id` images. These throw a clear endpoint-specific error. diff --git a/docs/advanced/multimodal-content.md b/docs/advanced/multimodal-content.md index c02f5fc89..ae394b581 100644 --- a/docs/advanced/multimodal-content.md +++ b/docs/advanced/multimodal-content.md @@ -260,7 +260,7 @@ const imagePart = { ### File Handle (Files API) -Use `type: 'file'` to reference media you uploaded once via a provider's [Files API](./files-api.md) — the provider stores the bytes and you pass a lightweight handle instead of re-sending base64 or a public URL every request. A handle only works with the provider that issued it, so the `provider` field is required and validated at request time. +Use `type: 'file'` to reference media you uploaded once via a provider's [Files API](./files-api.md) — the provider stores the bytes and you pass a lightweight reference instead of re-sending base64 or a public URL every request. The source carries a record of per-provider references (`{ reference: { openai: 'file-…' } }`); each adapter reads its own entry and throws when none is present, and adapters without Files API support reject the source before any request is built. ```typescript import { openaiFiles, openaiText } from '@tanstack/ai-openai' @@ -290,7 +290,7 @@ for await (const chunk of chat({ } ``` -`fileSourceFromHandle(handle)` builds the `{ type: 'file', value, provider }` source for you (picking the handle URL for Gemini/fal or the opaque id for OpenAI/Anthropic). Each adapter maps it to the provider's native reference (`file_id`, `fileData.fileUri`, or storage URL). Passing a handle to a different provider — or to an endpoint that requires raw bytes (image edits, Veo) — throws a clear error. See [Files API](./files-api.md) for uploading, retrieving, and deleting handles. +`fileSourceFromHandle(...handles)` builds the `{ type: 'file', reference }` source for you (picking the handle URL for Gemini/fal or the opaque id for OpenAI/Anthropic), and merges handles from several providers into one source that routes to any of them. Each adapter maps its own reference entry to the provider's native field (`file_id`, `fileData.fileUri`, or storage URL). Sending the source to a provider with no entry — or to an endpoint that requires raw bytes (image edits, Veo) — throws a clear error. See [Files API](./files-api.md) for uploading, retrieving, and deleting handles. ## Backward Compatibility diff --git a/examples/ts-react-chat/src/routes/index.tsx b/examples/ts-react-chat/src/routes/index.tsx index be30c2f91..501457ebd 100644 --- a/examples/ts-react-chat/src/routes/index.tsx +++ b/examples/ts-react-chat/src/routes/index.tsx @@ -334,8 +334,9 @@ function Messages({ ) } - // Render image parts - if (part.type === 'image') { + // Render image parts (file references have no local bytes + // or URL to render, so only url/data sources get an ) + if (part.type === 'image' && 'value' in part.source) { const imageUrl = part.source.type === 'url' ? part.source.value diff --git a/packages/ai-anthropic/src/adapters/text.ts b/packages/ai-anthropic/src/adapters/text.ts index d87099183..61b1bef59 100644 --- a/packages/ai-anthropic/src/adapters/text.ts +++ b/packages/ai-anthropic/src/adapters/text.ts @@ -1,6 +1,6 @@ import { EventType, - assertOwnFileSource, + fileReferenceFor, isFileSource, normalizeSystemPrompts, } from '@tanstack/ai' @@ -293,6 +293,8 @@ export class AnthropicTextAdapter< > { override readonly kind = 'text' as const readonly name = 'anthropic' as const + // Consumes `file_id` sources issued by anthropicFiles() (Files API beta). + override readonly supportsFileSources = true private readonly client: Anthropic_SDK @@ -668,8 +670,10 @@ export class AnthropicTextAdapter< | BetaURLImageSource | BetaFileImageSource if (isFileSource(part.source)) { - assertOwnFileSource(part.source, this.name) - imageSource = { type: 'file', file_id: part.source.value } + imageSource = { + type: 'file', + file_id: fileReferenceFor(part.source, this.name), + } } else if (part.source.type === 'data') { imageSource = { type: 'base64', @@ -699,8 +703,10 @@ export class AnthropicTextAdapter< | BetaURLPDFSource | BetaFileDocumentSource if (isFileSource(part.source)) { - assertOwnFileSource(part.source, this.name) - docSource = { type: 'file', file_id: part.source.value } + docSource = { + type: 'file', + file_id: fileReferenceFor(part.source, this.name), + } } else if (part.source.type === 'data') { docSource = { type: 'base64', diff --git a/packages/ai-anthropic/tests/files-source.test.ts b/packages/ai-anthropic/tests/files-source.test.ts index f176d04b4..a310eeb16 100644 --- a/packages/ai-anthropic/tests/files-source.test.ts +++ b/packages/ai-anthropic/tests/files-source.test.ts @@ -54,8 +54,7 @@ describe('anthropic file content source', () => { type: 'image', source: { type: 'file', - value: 'file_anthropic_123', - provider: 'anthropic', + reference: { anthropic: 'file_anthropic_123' }, }, }, ], @@ -89,8 +88,7 @@ describe('anthropic file content source', () => { type: 'image', source: { type: 'file', - value: 'file-openai-1', - provider: 'openai', + reference: { openai: 'file-openai-1' }, }, }, ], diff --git a/packages/ai-byteplus/tests/files-source.test.ts b/packages/ai-byteplus/tests/files-source.test.ts index 706c5bb97..438f07807 100644 --- a/packages/ai-byteplus/tests/files-source.test.ts +++ b/packages/ai-byteplus/tests/files-source.test.ts @@ -38,62 +38,51 @@ async function collectChunks( } describe('byteplus file content source', () => { - it('rejects a foreign provider file handle instead of sending it as a URL', async () => { - const chunks = await collectChunks( - chat({ - adapter: adapterWithMockClient(), - messages: [ - { - role: 'user', - content: [ - { - type: 'image', - source: { - type: 'file', - value: 'file-openai-abc', - provider: 'openai', + it('rejects a foreign provider file reference in core preflight, before any request', async () => { + await expect( + collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: [ + { + role: 'user', + content: [ + { + type: 'image', + source: { + type: 'file', + reference: { openai: 'file-openai-abc' }, + }, }, - }, - ], - }, - ], - }), - ) - - const runError = chunks.find((c) => c.type === 'RUN_ERROR') - expect(runError).toBeDefined() - if (runError?.type === 'RUN_ERROR') { - expect(runError.message).toMatch(/byteplus/) - expect(runError.message).toMatch(/file/) - } + ], + }, + ], + }), + ), + ).rejects.toThrow(/byteplus does not support provider file-handle/) }) - it('rejects a byteplus-marked file source — the provider has no Files API', async () => { - const chunks = await collectChunks( - chat({ - adapter: adapterWithMockClient(), - messages: [ - { - role: 'user', - content: [ - { - type: 'video', - source: { - type: 'file', - value: 'https://example.com/some-handle', - provider: 'byteplus', + it('rejects a byteplus-keyed file source — the provider has no Files API', async () => { + await expect( + collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: [ + { + role: 'user', + content: [ + { + type: 'video', + source: { + type: 'file', + reference: { byteplus: 'https://example.com/some-handle' }, + }, }, - }, - ], - }, - ], - }), - ) - - const runError = chunks.find((c) => c.type === 'RUN_ERROR') - expect(runError).toBeDefined() - if (runError?.type === 'RUN_ERROR') { - expect(runError.message).toMatch(/byteplus/) - } + ], + }, + ], + }), + ), + ).rejects.toThrow(/byteplus does not support provider file-handle/) }) }) diff --git a/packages/ai-event-client/src/index.ts b/packages/ai-event-client/src/index.ts index 21dbeb87a..53996059d 100644 --- a/packages/ai-event-client/src/index.ts +++ b/packages/ai-event-client/src/index.ts @@ -35,8 +35,8 @@ export interface ContentPartUrlSource { export interface ContentPartFileSource { type: 'file' - value: string - provider: string + /** Provider name → that provider's file reference (id or URI). */ + reference: Record mimeType?: string } diff --git a/packages/ai-fal/src/adapters/image.ts b/packages/ai-fal/src/adapters/image.ts index d0cfbd969..d0e8c0c64 100644 --- a/packages/ai-fal/src/adapters/image.ts +++ b/packages/ai-fal/src/adapters/image.ts @@ -53,6 +53,8 @@ export class FalImageAdapter extends BaseImageAdapter< > { override readonly kind = 'image' as const readonly name = 'fal' as const + // Consumes fal storage URLs uploaded via falFiles(). + override readonly supportsFileSources = true constructor(model: TModel, config?: FalClientConfig) { super(model, {}) diff --git a/packages/ai-fal/src/adapters/video.ts b/packages/ai-fal/src/adapters/video.ts index 4fffcdb5b..d0a9797ab 100644 --- a/packages/ai-fal/src/adapters/video.ts +++ b/packages/ai-fal/src/adapters/video.ts @@ -134,6 +134,8 @@ export class FalVideoAdapter extends BaseVideoAdapter< > { override readonly kind = 'video' as const readonly name = 'fal' as const + // Consumes fal storage URLs uploaded via falFiles(). + override readonly supportsFileSources = true constructor(model: TModel, config?: FalClientConfig) { super({}, model) diff --git a/packages/ai-fal/src/image/image-inputs.ts b/packages/ai-fal/src/image/image-inputs.ts index 692cf0237..7fe78a89a 100644 --- a/packages/ai-fal/src/image/image-inputs.ts +++ b/packages/ai-fal/src/image/image-inputs.ts @@ -1,4 +1,4 @@ -import { assertOwnFileSource, isFileSource } from '@tanstack/ai' +import { fileReferenceFor, isFileSource } from '@tanstack/ai' import { FAL_IMAGE_FIELD_OVERRIDES } from './generated/image-field-overrides' import type { FalImageFieldName, @@ -19,8 +19,9 @@ import type { FalModel, FalModelInput } from '../model-meta' */ export function contentSourceToFalUrl(source: ContentPartSource): string { if (isFileSource(source)) { - assertOwnFileSource(source, 'fal') - return source.value + // The 'fal' entry of the reference record is a storage URL; throws when + // the file was never uploaded to fal storage. + return fileReferenceFor(source, 'fal') } if (source.type === 'url') return source.value return `data:${source.mimeType};base64,${source.value}` diff --git a/packages/ai-fal/tests/content-source-to-fal-url.test.ts b/packages/ai-fal/tests/content-source-to-fal-url.test.ts index e8dbd01f4..0f922f4b9 100644 --- a/packages/ai-fal/tests/content-source-to-fal-url.test.ts +++ b/packages/ai-fal/tests/content-source-to-fal-url.test.ts @@ -6,8 +6,7 @@ describe('contentSourceToFalUrl', () => { expect( contentSourceToFalUrl({ type: 'file', - value: 'https://fal.media/files/abc.png', - provider: 'fal', + reference: { fal: 'https://fal.media/files/abc.png' }, }), ).toBe('https://fal.media/files/abc.png') }) @@ -16,8 +15,7 @@ describe('contentSourceToFalUrl', () => { expect(() => contentSourceToFalUrl({ type: 'file', - value: 'file-openai-123', - provider: 'openai', + reference: { openai: 'file-openai-123' }, }), ).toThrow(/fal/) }) diff --git a/packages/ai-gemini/src/adapters/image.ts b/packages/ai-gemini/src/adapters/image.ts index e7d9b2205..506daba4e 100644 --- a/packages/ai-gemini/src/adapters/image.ts +++ b/packages/ai-gemini/src/adapters/image.ts @@ -1,5 +1,5 @@ import { - assertOwnFileSource, + fileReferenceFor, isFileSource, resolveMediaPrompt, } from '@tanstack/ai' @@ -76,6 +76,8 @@ export class GeminiImageAdapter< > { override readonly kind = 'image' as const readonly name = 'gemini' as const + // Consumes Gemini Files API references (geminiFiles()) as fileData.fileUri. + override readonly supportsFileSources = true // Type-only property - never assigned at runtime declare '~types': { @@ -265,19 +267,18 @@ export class GeminiImageAdapter< }, } } - // A Gemini Files API handle from another provider is a bug — reject it - // before it's passed through as a fileData URI. - if (isFileSource(part.source)) { - assertOwnFileSource(part.source, this.name) - } // URL sources (public HTTPS, Files API URIs, gs://) pass through as // `fileData` and Gemini fetches them server-side — same as the chat // adapter. Fetching locally and inlining as base64 double-buffers the // image and OOMs on memory-constrained runtimes (e.g. Cloudflare - // Workers). + // Workers). A file source resolves to this adapter's own reference entry + // (throws when the file was never uploaded to Gemini). + const fileUri = isFileSource(part.source) + ? fileReferenceFor(part.source, this.name) + : part.source.value return { fileData: { - fileUri: part.source.value, + fileUri, mimeType: part.source.mimeType ?? 'image/jpeg', }, } diff --git a/packages/ai-gemini/src/adapters/text.ts b/packages/ai-gemini/src/adapters/text.ts index aadfab453..96cbaebd5 100644 --- a/packages/ai-gemini/src/adapters/text.ts +++ b/packages/ai-gemini/src/adapters/text.ts @@ -1,7 +1,7 @@ import { FinishReason } from '@google/genai' import { EventType, - assertOwnFileSource, + fileReferenceFor, isFileSource, normalizeSystemPrompts, } from '@tanstack/ai' @@ -116,6 +116,8 @@ export class GeminiTextAdapter< > { override readonly kind = 'text' as const readonly name = 'gemini' as const + // Consumes Gemini Files API references (geminiFiles()) as fileData.fileUri. + override readonly supportsFileSources = true private readonly client: GoogleGenAI @@ -667,12 +669,13 @@ export class GeminiTextAdapter< }, } } else { - // File handles (Gemini Files API) and public URLs both pass through as - // `fileData`; Gemini fetches the URI server-side. Reject a handle from - // another provider before it's sent. - if (isFileSource(part.source)) { - assertOwnFileSource(part.source, this.name) - } + // File references (Gemini Files API) and public URLs both pass + // through as `fileData`; Gemini fetches the URI server-side. A file + // source resolves to this adapter's own reference entry (throws when + // the file was never uploaded to Gemini). + const fileUri = isFileSource(part.source) + ? fileReferenceFor(part.source, this.name) + : part.source.value // For URL sources, use provided mimeType or fall back to reasonable defaults const defaultMimeType = { image: 'image/jpeg', @@ -683,7 +686,7 @@ export class GeminiTextAdapter< return { fileData: { - fileUri: part.source.value, + fileUri, mimeType: part.source.mimeType ?? defaultMimeType, }, } @@ -777,9 +780,9 @@ export class GeminiTextAdapter< }, }) } else { - if (isFileSource(part.source)) { - assertOwnFileSource(part.source, this.name) - } + const fileUri = isFileSource(part.source) + ? fileReferenceFor(part.source, this.name) + : part.source.value const defaultMimeType = { image: 'image/jpeg', audio: 'audio/mp3', @@ -788,7 +791,7 @@ export class GeminiTextAdapter< }[part.type] mediaParts.push({ fileData: { - fileUri: part.source.value, + fileUri, mimeType: part.source.mimeType ?? defaultMimeType, }, }) diff --git a/packages/ai-gemini/src/adapters/video.ts b/packages/ai-gemini/src/adapters/video.ts index d6e3bb6fc..72a0250a8 100644 --- a/packages/ai-gemini/src/adapters/video.ts +++ b/packages/ai-gemini/src/adapters/video.ts @@ -3,7 +3,7 @@ import { VideoGenerationReferenceType, } from '@google/genai' import { - assertOwnFileSource, + fileReferenceFor, isFileSource, resolveMediaPrompt, unsupportedFileSourceError, @@ -156,20 +156,21 @@ async function imagePartToVeoImage( function mediaPartToInteractionsContent( part: ImagePart | VideoPart, ): InteractionContent { - // A file handle from another provider is a bug; a Gemini handle maps to the - // `uri` field, same as a public URL (mirrors the Interactions text adapter). - if (isFileSource(part.source)) { - assertOwnFileSource(part.source, 'gemini') - } + // A Gemini Files API reference maps to the `uri` field, same as a public + // URL (mirrors the Interactions text adapter). `fileReferenceFor` throws + // when the file was never uploaded to Gemini. + const sourceValue = isFileSource(part.source) + ? fileReferenceFor(part.source, 'gemini') + : part.source.value const mimeType = part.source.mimeType if (part.type === 'image') { return part.source.type === 'data' - ? { type: 'image', data: part.source.value, mime_type: mimeType } - : { type: 'image', uri: part.source.value, mime_type: mimeType } + ? { type: 'image', data: sourceValue, mime_type: mimeType } + : { type: 'image', uri: sourceValue, mime_type: mimeType } } return part.source.type === 'data' - ? { type: 'video', data: part.source.value, mime_type: mimeType } - : { type: 'video', uri: part.source.value, mime_type: mimeType } + ? { type: 'video', data: sourceValue, mime_type: mimeType } + : { type: 'video', uri: sourceValue, mime_type: mimeType } } /** @@ -269,6 +270,9 @@ export class GeminiVideoAdapter< GeminiVideoModelDurationByName > { readonly name = 'gemini' as const + // The Interactions path consumes Gemini Files API references as content + // `uri`s; the Veo path still rejects them (raw bytes / gs:// only). + override readonly supportsFileSources = true protected client: GoogleGenAI private readonly allowUrlFetch: boolean diff --git a/packages/ai-gemini/src/experimental/text-interactions/adapter.ts b/packages/ai-gemini/src/experimental/text-interactions/adapter.ts index 2173f2874..422daba1a 100644 --- a/packages/ai-gemini/src/experimental/text-interactions/adapter.ts +++ b/packages/ai-gemini/src/experimental/text-interactions/adapter.ts @@ -1,4 +1,4 @@ -import { EventType, assertOwnFileSource, isFileSource } from '@tanstack/ai' +import { EventType, fileReferenceFor, isFileSource } from '@tanstack/ai' import { BaseTextAdapter } from '@tanstack/ai/adapters' import { parse as parsePartialJSON } from 'partial-json' import { @@ -175,6 +175,8 @@ export class GeminiTextInteractionsAdapter< > { override readonly kind = 'text' as const override readonly name = 'gemini-text-interactions' as const + // Consumes Gemini Files API references (geminiFiles()) as content `uri`s. + override readonly supportsFileSources = true private readonly client: GoogleGenAI // Tracks the most recent server-assigned interaction id per threadId @@ -728,11 +730,13 @@ function contentPartToBlock(part: ContentPart): ContentBlock { if (part.type === 'text') { return { type: 'text', text: part.content } } - // A file handle from another provider is a bug; a Gemini handle maps to the - // `uri` field (isData stays false), same as a public URL. - if (isFileSource(part.source)) { - assertOwnFileSource(part.source, 'gemini') - } + // A Gemini Files API reference maps to the `uri` field (isData stays + // false), same as a public URL. `fileReferenceFor` throws when the file was + // never uploaded to Gemini (the reference key is the files-issuer name, + // shared with the standard gemini adapters). + const sourceValue = isFileSource(part.source) + ? fileReferenceFor(part.source, 'gemini') + : part.source.value const isData = part.source.type === 'data' switch (part.type) { case 'image': { @@ -742,8 +746,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock { 'image', ) return isData - ? { type: 'image', data: part.source.value, mime_type } - : { type: 'image', uri: part.source.value, mime_type } + ? { type: 'image', data: sourceValue, mime_type } + : { type: 'image', uri: sourceValue, mime_type } } case 'audio': { const mime_type = validateMime( @@ -752,8 +756,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock { 'audio', ) return isData - ? { type: 'audio', data: part.source.value, mime_type } - : { type: 'audio', uri: part.source.value, mime_type } + ? { type: 'audio', data: sourceValue, mime_type } + : { type: 'audio', uri: sourceValue, mime_type } } case 'video': { const mime_type = validateMime( @@ -762,8 +766,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock { 'video', ) return isData - ? { type: 'video', data: part.source.value, mime_type } - : { type: 'video', uri: part.source.value, mime_type } + ? { type: 'video', data: sourceValue, mime_type } + : { type: 'video', uri: sourceValue, mime_type } } case 'document': { const mime_type = validateMime( @@ -772,8 +776,8 @@ function contentPartToBlock(part: ContentPart): ContentBlock { 'document', ) return isData - ? { type: 'document', data: part.source.value, mime_type } - : { type: 'document', uri: part.source.value, mime_type } + ? { type: 'document', data: sourceValue, mime_type } + : { type: 'document', uri: sourceValue, mime_type } } } } diff --git a/packages/ai-gemini/tests/files-source.test.ts b/packages/ai-gemini/tests/files-source.test.ts index 2f8bc927b..086afbf54 100644 --- a/packages/ai-gemini/tests/files-source.test.ts +++ b/packages/ai-gemini/tests/files-source.test.ts @@ -71,9 +71,10 @@ describe('gemini file content source', () => { type: 'image', source: { type: 'file', - value: - 'https://generativelanguage.googleapis.com/v1beta/files/abc-123', - provider: 'gemini', + reference: { + gemini: + 'https://generativelanguage.googleapis.com/v1beta/files/abc-123', + }, mimeType: 'image/png', }, }, @@ -109,8 +110,7 @@ describe('gemini file content source', () => { type: 'image', source: { type: 'file', - value: 'file-openai-1', - provider: 'openai', + reference: { openai: 'file-openai-1' }, }, }, ], diff --git a/packages/ai-grok/tests/files-source.test.ts b/packages/ai-grok/tests/files-source.test.ts index 60fa003ef..937966cfa 100644 --- a/packages/ai-grok/tests/files-source.test.ts +++ b/packages/ai-grok/tests/files-source.test.ts @@ -42,7 +42,10 @@ function fileSourceMessage(provider: string) { content: [ { type: 'image' as const, - source: { type: 'file' as const, value: 'file-abc', provider }, + source: { + type: 'file' as const, + reference: { [provider]: 'file-abc' }, + }, }, ], }, @@ -50,34 +53,25 @@ function fileSourceMessage(provider: string) { } describe('grok file content source', () => { - it('rejects an openai file handle instead of forwarding its file_id', async () => { - const chunks = await collectChunks( - chat({ - adapter: adapterWithMockClient(), - messages: fileSourceMessage('openai'), - }), - ) - - const runError = chunks.find((c) => c.type === 'RUN_ERROR') - expect(runError).toBeDefined() - if (runError?.type === 'RUN_ERROR') { - expect(runError.message).toMatch(/grok/) - expect(runError.message).toMatch(/file/) - } + it('rejects an openai file reference in core preflight, before any request', async () => { + await expect( + collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: fileSourceMessage('openai'), + }), + ), + ).rejects.toThrow(/grok does not support provider file-handle/) }) - it('rejects even a grok-marked file source — xAI has no Files API', async () => { - const chunks = await collectChunks( - chat({ - adapter: adapterWithMockClient(), - messages: fileSourceMessage('grok'), - }), - ) - - const runError = chunks.find((c) => c.type === 'RUN_ERROR') - expect(runError).toBeDefined() - if (runError?.type === 'RUN_ERROR') { - expect(runError.message).toMatch(/does not support provider file-handle/) - } + it('rejects even a grok-keyed file source — xAI has no Files API', async () => { + await expect( + collectChunks( + chat({ + adapter: adapterWithMockClient(), + messages: fileSourceMessage('grok'), + }), + ), + ).rejects.toThrow(/does not support provider file-handle/) }) }) diff --git a/packages/ai-openai/src/adapters/text.ts b/packages/ai-openai/src/adapters/text.ts index 6657b9c07..3d3062dc3 100644 --- a/packages/ai-openai/src/adapters/text.ts +++ b/packages/ai-openai/src/adapters/text.ts @@ -93,9 +93,10 @@ export class OpenAITextAdapter< override readonly kind = 'text' as const override readonly name = 'openai' as const // OpenAI's Responses endpoint consumes `file_id` references issued by its - // Files API (`openaiFiles()`); the openai-base default is false because - // compatible subclasses (Grok, Bedrock, custom) have no such surface. - protected override readonly supportsFileIdInput = true + // Files API (`openaiFiles()`). The default is undefined (unsupported) so + // compatible subclasses of the openai-base adapter (Grok, Bedrock, custom) + // — which have no such surface — fail closed in preflight. + override readonly supportsFileSources = true constructor(config: OpenAITextConfig, model: TModel) { super(model, 'openai', new OpenAI(config)) diff --git a/packages/ai-openai/tests/files-source.test.ts b/packages/ai-openai/tests/files-source.test.ts index d086c3c71..00a94598c 100644 --- a/packages/ai-openai/tests/files-source.test.ts +++ b/packages/ai-openai/tests/files-source.test.ts @@ -56,8 +56,7 @@ describe('openai file content source', () => { type: 'image', source: { type: 'file', - value: 'file-openai-abc', - provider: 'openai', + reference: { openai: 'file-openai-abc' }, }, }, ], @@ -93,8 +92,7 @@ describe('openai file content source', () => { type: 'document', source: { type: 'file', - value: 'file-openai-pdf', - provider: 'openai', + reference: { openai: 'file-openai-pdf' }, }, }, ], @@ -162,8 +160,7 @@ describe('openai file content source', () => { type: 'image', source: { type: 'file', - value: 'files/gemini-xyz', - provider: 'gemini', + reference: { gemini: 'files/gemini-xyz' }, }, }, ], diff --git a/packages/ai-openrouter/src/adapters/responses-text.ts b/packages/ai-openrouter/src/adapters/responses-text.ts index f95cccf33..b656c7693 100644 --- a/packages/ai-openrouter/src/adapters/responses-text.ts +++ b/packages/ai-openrouter/src/adapters/responses-text.ts @@ -1685,23 +1685,34 @@ export class OpenRouterResponsesTextAdapter< protected convertContentPartToInput( part: ContentPart, ): ResponsesInputContent { - if ('source' in part && isFileSource(part.source)) { + if (part.type === 'text') { + return { + type: 'input_text', + text: part.content, + } + } + // Narrow once so the branches below can only see url/data sources — a + // `{ type: 'file' }` reference has no `value` to mis-map. A part without + // a source (unknown/malformed type) is rejected the same way as the + // default branch below. + const source = (part as { source?: typeof part.source }).source + if (source === undefined) { + throw new Error( + `Unsupported content part type for ${this.name}: ${(part as { type: string }).type}`, + ) + } + if (isFileSource(source)) { throw unsupportedFileSourceError(this.name) } switch (part.type) { - case 'text': - return { - type: 'input_text', - text: part.content, - } case 'image': { const meta = part.metadata as | { detail?: 'auto' | 'low' | 'high' } | undefined - const value = part.source.value + const value = source.value const imageUrl = - part.source.type === 'data' && !value.startsWith('data:') - ? `data:${part.source.mimeType || 'application/octet-stream'};base64,${value}` + source.type === 'data' && !value.startsWith('data:') + ? `data:${source.mimeType || 'application/octet-stream'};base64,${value}` : value return { type: 'input_image', @@ -1710,36 +1721,36 @@ export class OpenRouterResponsesTextAdapter< } } case 'audio': { - if (part.source.type === 'url') { + if (source.type === 'url') { // OpenRouter's `input_audio` carries `{ data, format }` not a URL — // fall back to `input_file` for URLs so we don't silently drop the // audio reference. return { type: 'input_file', - fileUrl: part.source.value, + fileUrl: source.value, } } return { type: 'input_audio', - inputAudio: { data: part.source.value, format: 'mp3' }, + inputAudio: { data: source.value, format: 'mp3' }, } } case 'video': return { type: 'input_video', - videoUrl: part.source.value, + videoUrl: source.value, } case 'document': { - if (part.source.type === 'url') { + if (source.type === 'url') { return { type: 'input_file', - fileUrl: part.source.value, + fileUrl: source.value, } } - const mime = part.source.mimeType || 'application/octet-stream' - const data = part.source.value.startsWith('data:') - ? part.source.value - : `data:${mime};base64,${part.source.value}` + const mime = source.mimeType || 'application/octet-stream' + const data = source.value.startsWith('data:') + ? source.value + : `data:${mime};base64,${source.value}` return { type: 'input_file', fileData: data, diff --git a/packages/ai-openrouter/src/adapters/text.ts b/packages/ai-openrouter/src/adapters/text.ts index b7c9cd43c..586642e4b 100644 --- a/packages/ai-openrouter/src/adapters/text.ts +++ b/packages/ai-openrouter/src/adapters/text.ts @@ -1302,22 +1302,29 @@ export class OpenRouterTextAdapter< /** OpenRouter content-part converter (camelCase imageUrl/inputAudio/videoUrl). */ protected convertContentPart(part: ContentPart): ChatContentItems | null { - if ('source' in part && isFileSource(part.source)) { + if (part.type === 'text') { + return { type: 'text', text: part.content } + } + // Narrow once so the branches below can only see url/data sources — a + // `{ type: 'file' }` reference has no `value` to mis-map. A part without + // a source (unknown/malformed type) falls through to the base's + // unsupported-content-part guard, matching the old default branch. + const source = (part as { source?: typeof part.source }).source + if (source === undefined) return null + if (isFileSource(source)) { throw unsupportedFileSourceError(this.name) } switch (part.type) { - case 'text': - return { type: 'text', text: part.content } case 'image': { const meta = part.metadata as OpenRouterImageMetadata | undefined - const value = part.source.value + const value = source.value // Default to `application/octet-stream` when the source didn't // provide a MIME type — interpolating `undefined` into the URI // ("data:undefined;base64,...") produces an invalid data URI the // API rejects. - const imageMime = part.source.mimeType || 'application/octet-stream' + const imageMime = source.mimeType || 'application/octet-stream' const url = - part.source.type === 'data' && !value.startsWith('data:') + source.type === 'data' && !value.startsWith('data:') ? `data:${imageMime};base64,${value}` : value return { @@ -1333,32 +1340,32 @@ export class OpenRouterTextAdapter< // base64 slot. The Responses adapter does have an `input_file` // URL variant and routes URLs there directly — see // `responses-text.ts`. - if (part.source.type === 'url') { + if (source.type === 'url') { return { type: 'text', - text: `[Audio: ${part.source.value}]`, + text: `[Audio: ${source.value}]`, } } return { type: 'input_audio', - inputAudio: { data: part.source.value, format: 'mp3' }, + inputAudio: { data: source.value, format: 'mp3' }, } case 'video': return { type: 'video_url', - videoUrl: { url: part.source.value }, + videoUrl: { url: source.value }, } case 'document': // The chat-completions SDK has no document_url type. For URL // sources, surface a text reference so the model at least sees - // the link. For data sources, `part.source.value` is the raw + // the link. For data sources, `source.value` is the raw // base64 payload — inlining it into the prompt would blow the // context window with megabytes of binary and leak the document // content verbatim. Throw instead so the caller can either // switch to the Responses adapter (which has proper input_file // support for data documents) or strip the document before // sending. - if (part.source.type === 'data') { + if (source.type === 'data') { throw new Error( `${this.name} chat-completions does not support inline (data) document content parts. ` + `Use the Responses adapter (openRouterResponsesText) for document data, ` + @@ -1367,7 +1374,7 @@ export class OpenRouterTextAdapter< } return { type: 'text', - text: `[Document: ${part.source.value}]`, + text: `[Document: ${source.value}]`, } default: return null diff --git a/packages/ai/skills/ai-core/adapter-configuration/SKILL.md b/packages/ai/skills/ai-core/adapter-configuration/SKILL.md index 76c90919e..33cc21836 100644 --- a/packages/ai/skills/ai-core/adapter-configuration/SKILL.md +++ b/packages/ai/skills/ai-core/adapter-configuration/SKILL.md @@ -452,17 +452,24 @@ chat({ Rules agents must respect: -- **A handle only works with the provider that issued it.** Adapters validate - `source.provider` at request time and throw on a mismatch; the `FileHandle` - provider-literal types also reject cross-provider `getFile()`/`deleteFile()` - calls at compile time. +- **The source is a per-provider reference record.** `fileSourceFromHandle` + builds `{ type: 'file', reference: { openai: 'file-…' } }`; each adapter + reads only its own entry and throws when none is present. Upload the same + bytes to several providers and pass all the handles — + `fileSourceFromHandle(openaiHandle, geminiHandle)` — to build one source + that routes to any of them. +- **Adapters declare `supportsFileSources`.** For adapters that don't (Grok, + Groq, Bedrock, Mistral, OpenRouter, Ollama, BytePlus, and anything written + before this feature), `chat()` / `generateImage()` / `generateVideo()` + reject file sources in preflight, before any request is built — pass + `data`/`url` sources there instead. - **Lifecycle:** `getFile()` / `deleteFile()` work for OpenAI, Anthropic, and - Gemini. fal storage is upload-only — those calls throw for `falFiles()`. -- **Not every endpoint consumes handles.** Chat Completions image inputs, - OpenAI `images/edits` + Sora `input_reference`, Gemini Veo, and providers - without a Files API (Grok, Groq, Bedrock, Mistral, OpenRouter, Ollama, - BytePlus) throw a clear "unsupported file source" error — pass `data`/`url` - sources there instead. + Gemini, and accept the handle itself (provider-literal typed — a foreign + handle is a compile error). fal storage is upload-only — those calls throw + for `falFiles()`. +- **Some endpoints need raw bytes even on supporting providers:** OpenAI + `images/edits` + Sora `input_reference`, Gemini Veo, and Chat Completions + image inputs throw endpoint-specific errors for file sources. - `fileSourceFromHandle` and the `FileHandle` type are also exported from the browser-safe `@tanstack/ai/client` entry for clients that persist handles. diff --git a/packages/ai/skills/ai-core/chat-experience/SKILL.md b/packages/ai/skills/ai-core/chat-experience/SKILL.md index 181e561db..642f6f552 100644 --- a/packages/ai/skills/ai-core/chat-experience/SKILL.md +++ b/packages/ai/skills/ai-core/chat-experience/SKILL.md @@ -292,10 +292,13 @@ if (part.type === 'image') { For media reused across turns, upload once via a provider Files adapter (`openaiFiles()`, `anthropicFiles()`, `geminiFiles()`, `falFiles()`) and send a -`{ type: 'file' }` source built with `fileSourceFromHandle(handle)` instead of -re-sending base64 each request. `fileSourceFromHandle` is exported from the -browser-safe `@tanstack/ai/client`; the upload itself is server-side. A handle -only works with the provider that issued it. See +`{ type: 'file' }` source built with `fileSourceFromHandle(...handles)` instead +of re-sending base64 each request. The source carries a per-provider reference +record, so handles from several providers merge into one source that routes to +any of them; providers without an entry (or without Files API support at all) +reject it with a clear error before any request is sent. +`fileSourceFromHandle` is exported from the browser-safe +`@tanstack/ai/client`; the upload itself is server-side. See `ai-core/adapter-configuration/SKILL.md` §7 and `docs/advanced/files-api.md`. ### 4. Sending Audio Messages (Browser Recording) diff --git a/packages/ai/src/activities/chat/adapter.ts b/packages/ai/src/activities/chat/adapter.ts index cb3bcae6d..fe13e1d60 100644 --- a/packages/ai/src/activities/chat/adapter.ts +++ b/packages/ai/src/activities/chat/adapter.ts @@ -89,6 +89,15 @@ export interface TextAdapter< */ readonly requires?: ReadonlyArray + /** + * Declares that this adapter can consume `{ type: 'file' }` content sources + * (provider Files API references). `chat()` rejects file sources in preflight + * for adapters that don't declare this, so an adapter written before the + * file arm existed fails closed instead of silently mis-mapping a reference + * onto its URL/data branch. + */ + readonly supportsFileSources?: boolean + /** * @internal Type-only properties for inference. Not assigned at runtime. */ @@ -194,6 +203,7 @@ export abstract class BaseTextAdapter< abstract readonly name: string readonly model: TModel readonly requires?: ReadonlyArray = undefined + readonly supportsFileSources?: boolean = undefined // Type-only property - never assigned at runtime declare '~types': { diff --git a/packages/ai/src/activities/chat/index.ts b/packages/ai/src/activities/chat/index.ts index b884d5449..78723b487 100644 --- a/packages/ai/src/activities/chat/index.ts +++ b/packages/ai/src/activities/chat/index.ts @@ -24,6 +24,7 @@ import { } from '../../interrupt-serialization' import { normalizeToolResult } from '../../utilities/tool-result' import { isProviderExecutedToolCall } from '../../utilities/provider-executed' +import { assertMessagesFileSourceSupport } from '../../utilities/content-source' import { LazyToolManager } from './tools/lazy-tool-manager' import { MiddlewareAbortError, @@ -1335,6 +1336,13 @@ class TextEngine< ) } + // Fail closed on `{ type: 'file' }` sources for adapters that haven't + // declared support — an adapter written before the file arm existed would + // otherwise fall through to its URL/data branch and silently mis-map the + // reference. Checked per model call so tool results added mid-loop are + // covered too. + assertMessagesFileSourceSupport(this.adapter, this.messages) + for await (const chunk of this.adapter.chatStream({ model: this.params.model, messages: this.messages, diff --git a/packages/ai/src/activities/files/index.ts b/packages/ai/src/activities/files/index.ts index dc6b800a5..2514b0ac5 100644 --- a/packages/ai/src/activities/files/index.ts +++ b/packages/ai/src/activities/files/index.ts @@ -82,27 +82,37 @@ export async function deleteFile(options: { } /** - * Build a `{ type: 'file' }` content source from an uploaded {@link FileHandle}, - * for use in a chat message (image/audio/document part `source`). + * Build a `{ type: 'file' }` content source from one or more uploaded + * {@link FileHandle}s, for use in a chat message (image/audio/document part + * `source`). * - * Picks the right `value`: the handle URL when the provider exposes one - * (Gemini/fal), otherwise the opaque id (OpenAI/Anthropic). + * Each handle contributes a `reference` entry under its provider name, using + * the right wire form: the handle URL when the provider exposes one + * (Gemini/fal), otherwise the opaque id (OpenAI/Anthropic). Pass handles from + * several providers (the same bytes uploaded to each) to build a source that + * routes correctly to any of them. * * @example * ```ts - * const handle = await uploadFile({ adapter: openaiFiles(), input }) + * const openaiHandle = await uploadFile({ adapter: openaiFiles(), input }) + * const geminiHandle = await uploadFile({ adapter: geminiFiles(), input }) * messages.push({ role: 'user', content: [ - * { type: 'image', source: fileSourceFromHandle(handle) }, + * { type: 'image', source: fileSourceFromHandle(openaiHandle, geminiHandle) }, * ] }) * ``` */ export function fileSourceFromHandle( - handle: FileHandle, + ...handles: [FileHandle, ...Array>] ): ContentPartFileSource { + const reference = {} as Record + let mimeType: string | undefined + for (const handle of handles) { + reference[handle.provider] = handle.uri ?? handle.id + mimeType ??= handle.mimeType + } return { type: 'file', - value: handle.uri ?? handle.id, - provider: handle.provider, - ...(handle.mimeType ? { mimeType: handle.mimeType } : {}), + reference, + ...(mimeType ? { mimeType } : {}), } } diff --git a/packages/ai/src/activities/generateImage/adapter.ts b/packages/ai/src/activities/generateImage/adapter.ts index cbc24b72c..e97c40939 100644 --- a/packages/ai/src/activities/generateImage/adapter.ts +++ b/packages/ai/src/activities/generateImage/adapter.ts @@ -51,6 +51,13 @@ export interface ImageAdapter< readonly kind: 'image' /** Adapter name identifier */ readonly name: string + /** + * Declares that this adapter can consume `{ type: 'file' }` content + * sources (provider Files API references). The activity dispatcher rejects + * file sources in preflight for adapters that don't declare this, so + * adapters written before the file arm existed fail closed. + */ + readonly supportsFileSources?: boolean /** The model this adapter is configured for */ readonly model: TModel @@ -103,6 +110,7 @@ export abstract class BaseImageAdapter< > { readonly kind = 'image' as const abstract readonly name: string + readonly supportsFileSources?: boolean = undefined readonly model: TModel // Type-only property - never assigned at runtime diff --git a/packages/ai/src/activities/generateImage/index.ts b/packages/ai/src/activities/generateImage/index.ts index 336be0842..2b8f9e68d 100644 --- a/packages/ai/src/activities/generateImage/index.ts +++ b/packages/ai/src/activities/generateImage/index.ts @@ -17,6 +17,7 @@ import { runGenerationUsage, } from '../middleware/run' import { resolveMediaPrompt } from '../../utilities/media-prompt' +import { assertPromptFileSourceSupport } from '../../utilities/content-source' import type { InternalLogger } from '../../logger/internal-logger' import type { DebugOption } from '../../logger/types' import type { GenerationMiddleware } from '../middleware/types' @@ -281,6 +282,10 @@ async function runGenerateImage< await runGenerationStart(middleware, mwCtx) + // Fail closed on `{ type: 'file' }` sources for adapters that haven't + // declared support (see assertPromptFileSourceSupport). + assertPromptFileSourceSupport(adapter, rest.prompt) + // Devtools events carry the flattened prompt text plus media-part counts — // the wire payload stays `prompt: string` regardless of the prompt shape. const resolved = resolveMediaPrompt(rest.prompt) diff --git a/packages/ai/src/activities/generateVideo/adapter.ts b/packages/ai/src/activities/generateVideo/adapter.ts index 64dd0162e..afd4eaa18 100644 --- a/packages/ai/src/activities/generateVideo/adapter.ts +++ b/packages/ai/src/activities/generateVideo/adapter.ts @@ -74,6 +74,13 @@ export interface VideoAdapter< readonly kind: 'video' /** Adapter name identifier */ readonly name: string + /** + * Declares that this adapter can consume `{ type: 'file' }` content + * sources (provider Files API references). The activity dispatcher rejects + * file sources in preflight for adapters that don't declare this, so + * adapters written before the file arm existed fail closed. + */ + readonly supportsFileSources?: boolean /** The model this adapter is configured for */ readonly model: TModel @@ -161,6 +168,7 @@ export abstract class BaseVideoAdapter< > { readonly kind = 'video' as const abstract readonly name: string + readonly supportsFileSources?: boolean = undefined readonly model: TModel // Type-only property - never assigned at runtime diff --git a/packages/ai/src/activities/generateVideo/index.ts b/packages/ai/src/activities/generateVideo/index.ts index 433eb79b0..3b21e0925 100644 --- a/packages/ai/src/activities/generateVideo/index.ts +++ b/packages/ai/src/activities/generateVideo/index.ts @@ -10,6 +10,7 @@ import { aiEventClient } from '@tanstack/ai-event-client' import { toRunErrorPayload } from '../error-payload' import { resolveDebugOption } from '../../logger/resolve' +import { assertPromptFileSourceSupport } from '../../utilities/content-source' import { applyGenerationResultTransforms, createGenerationContext, @@ -414,6 +415,9 @@ async function runCreateVideoJob< TAdapter extends VideoAdapter, >(options: VideoCreateOptions): Promise { const { adapter, prompt, size, duration, modelOptions, middleware } = options + // Fail closed on `{ type: 'file' }` sources for adapters that haven't + // declared support (see assertPromptFileSourceSupport). + assertPromptFileSourceSupport(adapter, prompt) const model = adapter.model const requestId = createId('video') const startTime = Date.now() @@ -501,6 +505,9 @@ async function* runStreamingVideoGeneration< TAdapter extends VideoAdapter, >(options: VideoCreateOptions): AsyncIterable { const { adapter, prompt, size, duration, modelOptions, middleware } = options + // Fail closed on `{ type: 'file' }` sources for adapters that haven't + // declared support (see assertPromptFileSourceSupport). + assertPromptFileSourceSupport(adapter, prompt) const model = adapter.model const runId = options.runId ?? createId('run') const requestId = createId('video') diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 1b6659770..4f91a450d 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -404,9 +404,12 @@ export { normalizeToolResult, } from './utilities/tool-result' export { - assertOwnFileSource, + assertMessagesFileSourceSupport, + assertPromptFileSourceSupport, + fileReferenceFor, isFileSource, unsupportedFileSourceError, + type FileSourceCapable, } from './utilities/content-source' export { diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 22efea533..5804e39ca 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -245,33 +245,31 @@ export interface ContentPartUrlSource { } /** - * Source specification for a provider-issued file handle (Files API). + * Source specification for provider-issued file references (Files API). * * The media is uploaded once via a `files` adapter (`openaiFiles()`, * `anthropicFiles()`, `geminiFiles()`, `falFiles()`) and referenced here by the * returned handle instead of re-sending base64 or a public URL each request. * - * A handle only routes to the provider that issued it — the `provider` field is - * validated at map time, and adapters throw if it doesn't match. The `value` is - * the provider's opaque id (OpenAI/Anthropic `file_id`) or handle URL (Gemini - * file URI, fal storage URL); use `fileSourceFromHandle` to build one from a - * `FileHandle` without worrying about the id-vs-uri distinction. + * `reference` maps provider names to that provider's wire reference — an + * OpenAI/Anthropic `file_id`, a Gemini file URI, a fal storage URL. Upload the + * same bytes to several providers and merge their handles + * (`fileSourceFromHandle(openaiHandle, geminiHandle)`) to make one source that + * works across all of them; each adapter reads only its own entry and throws + * when none is present. Adapters that can't consume file references at all are + * rejected by the activity-layer preflight before mapping starts. */ export interface ContentPartFileSource { /** - * Indicates this references a provider-issued file handle. + * Indicates this references provider-issued file handles. */ type: 'file' /** - * The provider handle: an OpenAI/Anthropic `file_id`, a Gemini file URI, or a - * fal storage URL. + * Provider name → wire reference issued by that provider's Files API. Use + * `fileSourceFromHandle(...handles)` to build (and merge) entries without + * worrying about the id-vs-uri distinction. */ - value: string - /** - * The provider that issued the handle. A handle is only valid for its issuer; - * passing (e.g.) an OpenAI `file-...` id to a Gemini adapter is an error. - */ - provider: TProvider + reference: Record /** * Optional MIME type hint for cases where the provider can't infer it. */ diff --git a/packages/ai/src/utilities/content-source.ts b/packages/ai/src/utilities/content-source.ts index e7bac5615..4106ec168 100644 --- a/packages/ai/src/utilities/content-source.ts +++ b/packages/ai/src/utilities/content-source.ts @@ -1,12 +1,12 @@ import type { ContentPartFileSource, ContentPartSource } from '../types' /** - * Narrow a {@link ContentPartSource} to the provider-file-handle arm. + * Narrow a {@link ContentPartSource} to the provider-file-reference arm. * - * Every adapter that maps a content part's `source` onto a provider wire format - * must handle `{ type: 'file' }` explicitly — either mapping it to the provider's - * native file-reference field (issuers) or rejecting it (everyone else). Using - * this guard keeps that branch consistent across the ~dozen adapter packages. + * Issuer adapters use this to route a file source to their native wire field; + * everyone else is protected by the core preflight (see + * {@link assertMessagesFileSourceSupport}) plus a defensive throw at their own + * mapping site. */ export function isFileSource( source: ContentPartSource, @@ -15,37 +15,44 @@ export function isFileSource( } /** - * Assert that a file source's handle was issued by `providerName`. A provider - * file handle is only valid for the provider that created it (an OpenAI - * `file-...` id sent to Gemini is a bug), so issuer adapters call this before - * mapping the handle onto their wire format. + * Resolve the wire reference `providerName` should send for a file source. * - * @throws if `source.provider` doesn't match `providerName`. + * A file source carries a record of per-provider references (`{ openai: + * 'file-abc', gemini: 'https://…' }`) — upload the same bytes to several + * providers and merge their handles to make one source usable across all of + * them. An adapter only ever reads its own entry. + * + * @throws when the record has no entry for `providerName` — the file was + * never uploaded to this provider. */ -export function assertOwnFileSource( +export function fileReferenceFor( source: ContentPartFileSource, providerName: string, -): void { - if (source.provider !== providerName) { +): string { + const reference = source.reference[providerName] + if (reference === undefined) { + const available = Object.keys(source.reference) throw new Error( - `${providerName}: file source references a handle issued by ` + - `"${source.provider}" — a provider file handle only works with the ` + - `provider that created it. Upload the file with ${providerName}Files() ` + - `and reference that handle, or pass a data/url source instead.`, + `${providerName}: file source has no reference for this provider ` + + `(found: ${available.length > 0 ? available.join(', ') : 'none'}). ` + + `A provider file reference only works with the provider that issued ` + + `it — upload the file with ${providerName}Files() and merge that ` + + `handle into the source, or pass a data/url source instead.`, ) } + return reference } /** * Build the standard error a non-issuer adapter throws when it encounters a * `{ type: 'file' }` source it can't consume — either because the provider has - * no file-handle input surface, or because the endpoint requires raw bytes + * no file-reference input surface, or because the endpoint requires raw bytes * (image edits, Veo) rather than a reference. * * @param detail Optional context appended to the message (e.g. a modality or - * endpoint name, or a pointer to the adapter that does support handles). When - * provided it replaces the generic remediation tail, so a site-specific hint - * ("pass inline bytes") is never contradicted by generic advice. + * endpoint name, or a pointer to the adapter that does support references). + * When provided it replaces the generic remediation tail, so a site-specific + * hint ("pass inline bytes") is never contradicted by generic advice. */ export function unsupportedFileSourceError( providerName: string, @@ -60,3 +67,68 @@ export function unsupportedFileSourceError( `adapter where supported.`), ) } + +/** + * The slice of an adapter the file-source preflight reads. Adapters that can + * consume `{ type: 'file' }` sources declare `supportsFileSources: true`; + * everything else — including adapters written before this arm existed — + * fails closed at the activity layer instead of falling through to a + * URL/data branch and silently mis-mapping the reference. + */ +export interface FileSourceCapable { + name: string + supportsFileSources?: boolean +} + +/** True when a content-part-like value carries a `{ type: 'file' }` source. */ +function partHasFileSource(part: unknown): boolean { + if (typeof part !== 'object' || part === null) return false + const source = (part as { source?: unknown }).source + return ( + typeof source === 'object' && + source !== null && + (source as { type?: unknown }).type === 'file' + ) +} + +/** + * Fail-closed preflight for media prompts (`generateImage` / `generateVideo` / + * `generateAudio`): throws when the prompt carries a `{ type: 'file' }` source + * and the adapter hasn't declared `supportsFileSources`. Runs in the activity + * dispatcher — the same layer that validates modality — so an adapter that + * predates the file arm can never receive one. + */ +export function assertPromptFileSourceSupport( + adapter: FileSourceCapable, + prompt: unknown, +): void { + if (adapter.supportsFileSources === true) return + if (!Array.isArray(prompt)) return + for (const part of prompt) { + if (partHasFileSource(part)) { + throw unsupportedFileSourceError(adapter.name) + } + } +} + +/** + * Fail-closed preflight for chat messages: throws when any message content + * part carries a `{ type: 'file' }` source and the adapter hasn't declared + * `supportsFileSources`. See {@link assertPromptFileSourceSupport}. + */ +export function assertMessagesFileSourceSupport( + adapter: FileSourceCapable, + messages: ReadonlyArray, +): void { + if (adapter.supportsFileSources === true) return + for (const message of messages) { + if (typeof message !== 'object' || message === null) continue + const content = (message as { content?: unknown }).content + if (!Array.isArray(content)) continue + for (const part of content) { + if (partHasFileSource(part)) { + throw unsupportedFileSourceError(adapter.name) + } + } + } +} diff --git a/packages/ai/src/utilities/tool-result.ts b/packages/ai/src/utilities/tool-result.ts index 5a18039ec..e9bc85f94 100644 --- a/packages/ai/src/utilities/tool-result.ts +++ b/packages/ai/src/utilities/tool-result.ts @@ -25,13 +25,19 @@ export function isContentPart(value: unknown): value is ContentPart { const source = part.source if (typeof source !== 'object' || source === null) return false const src = source as Record + // `file` sources carry a non-empty provider→reference record instead of a + // `value` string. + if (src.type === 'file') { + const reference = src.reference + if (typeof reference !== 'object' || reference === null) return false + const entries = Object.values(reference) + return entries.length > 0 && entries.every((v) => typeof v === 'string') + } if (typeof src.value !== 'string') return false // `data` sources require a mimeType (matches ContentPartDataSource); `url` // sources don't. Requiring it here keeps the runtime guard consistent with // the type and avoids emitting `data:undefined;base64,...` downstream. if (src.type === 'data') return typeof src.mimeType === 'string' - // `file` sources reference a provider-issued handle and must name their issuer. - if (src.type === 'file') return typeof src.provider === 'string' return src.type === 'url' } diff --git a/packages/ai/tests/files-source.test.ts b/packages/ai/tests/files-source.test.ts index 691dcc8aa..14c7bc79c 100644 --- a/packages/ai/tests/files-source.test.ts +++ b/packages/ai/tests/files-source.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest' import { - assertOwnFileSource, + assertMessagesFileSourceSupport, + assertPromptFileSourceSupport, deleteFile, + fileReferenceFor, fileSourceFromHandle, getFile, isContentPart, @@ -15,8 +17,7 @@ import type { FileHandle, FilesAdapter } from '../src/activities/files/adapter' const fileSource: ContentPartSource = { type: 'file', - value: 'file-abc', - provider: 'openai', + reference: { openai: 'file-abc' }, } describe('file content source helpers', () => { @@ -28,9 +29,17 @@ describe('file content source helpers', () => { ).toBe(false) }) - it('assertOwnFileSource passes on match and throws on mismatch', () => { - expect(() => assertOwnFileSource(fileSource, 'openai')).not.toThrow() - expect(() => assertOwnFileSource(fileSource, 'gemini')).toThrow(/openai/) + it('fileReferenceFor resolves the own-provider entry and throws on a miss', () => { + const merged: ContentPartSource = { + type: 'file', + reference: { openai: 'file-abc', gemini: 'https://g/files/xyz' }, + } + if (!isFileSource(merged)) throw new Error('expected file source') + expect(fileReferenceFor(merged, 'openai')).toBe('file-abc') + expect(fileReferenceFor(merged, 'gemini')).toBe('https://g/files/xyz') + expect(() => fileReferenceFor(merged, 'anthropic')).toThrow( + /anthropic.*found: openai, gemini/s, + ) }) it('unsupportedFileSourceError includes provider and detail', () => { @@ -39,12 +48,11 @@ describe('file content source helpers', () => { expect(err.message).toContain('on this endpoint') }) - it('fileSourceFromHandle prefers uri (Gemini/fal), else id (OpenAI/Anthropic)', () => { + it('fileSourceFromHandle uses uri (Gemini/fal) else id (OpenAI/Anthropic) and merges handles', () => { const opaque: FileHandle = { id: 'file-abc', provider: 'openai' } expect(fileSourceFromHandle(opaque)).toEqual({ type: 'file', - value: 'file-abc', - provider: 'openai', + reference: { openai: 'file-abc' }, }) const withUri: FileHandle = { @@ -55,20 +63,75 @@ describe('file content source helpers', () => { } expect(fileSourceFromHandle(withUri)).toEqual({ type: 'file', - value: 'https://generativelanguage.googleapis.com/v1/files/xyz', - provider: 'gemini', + reference: { + gemini: 'https://generativelanguage.googleapis.com/v1/files/xyz', + }, + mimeType: 'image/png', + }) + + // Multiple handles (same bytes uploaded to two providers) merge into one + // source that routes to either provider. + expect(fileSourceFromHandle(opaque, withUri)).toEqual({ + type: 'file', + reference: { + openai: 'file-abc', + gemini: 'https://generativelanguage.googleapis.com/v1/files/xyz', + }, mimeType: 'image/png', }) }) - it('isContentPart accepts a valid file source and rejects one missing provider', () => { + it('isContentPart accepts a valid file source and rejects an empty reference record', () => { expect(isContentPart({ type: 'image', source: fileSource })).toBe(true) expect( isContentPart({ type: 'image', - source: { type: 'file', value: 'file-abc' }, + source: { type: 'file', reference: {} }, }), ).toBe(false) + expect( + isContentPart({ + type: 'image', + source: { type: 'file' }, + }), + ).toBe(false) + }) +}) + +describe('file source preflight', () => { + const fileMessage = { + role: 'user', + content: [{ type: 'image', source: fileSource }], + } + const plainMessage = { role: 'user', content: 'hello' } + + it('rejects file sources for adapters that do not declare support', () => { + expect(() => + assertMessagesFileSourceSupport({ name: 'legacy' }, [ + plainMessage, + fileMessage, + ]), + ).toThrow(/legacy does not support provider file-handle sources/) + expect(() => + assertPromptFileSourceSupport({ name: 'legacy' }, [ + { type: 'image', source: fileSource }, + ]), + ).toThrow(/legacy does not support provider file-handle sources/) + }) + + it('passes when the adapter declares support or no file source is present', () => { + expect(() => + assertMessagesFileSourceSupport( + { name: 'openai', supportsFileSources: true }, + [fileMessage], + ), + ).not.toThrow() + expect(() => + assertMessagesFileSourceSupport({ name: 'legacy' }, [plainMessage]), + ).not.toThrow() + expect(() => + assertPromptFileSourceSupport({ name: 'legacy' }, 'a text prompt'), + ).not.toThrow() }) }) diff --git a/packages/ai/tests/media-prompt.test.ts b/packages/ai/tests/media-prompt.test.ts index 18bd1dc12..01a912ac5 100644 --- a/packages/ai/tests/media-prompt.test.ts +++ b/packages/ai/tests/media-prompt.test.ts @@ -34,10 +34,11 @@ describe('resolveMediaPrompt', () => { const resolved = resolveMediaPrompt(parts) expect(resolved.text).toBe('animate this') expect(resolved.parts).toBe(parts) - expect(resolved.images.map((p) => p.source.value)).toEqual([ - 'https://a.png', - 'https://b.png', - ]) + expect( + resolved.images.map((p) => + 'value' in p.source ? p.source.value : undefined, + ), + ).toEqual(['https://a.png', 'https://b.png']) expect(resolved.images[1]?.metadata?.role).toBe('end_frame') expect(resolved.videos).toHaveLength(1) expect(resolved.audios).toHaveLength(1) diff --git a/packages/openai-base/src/adapters/responses-text.ts b/packages/openai-base/src/adapters/responses-text.ts index a03ff9205..3d56e6a88 100644 --- a/packages/openai-base/src/adapters/responses-text.ts +++ b/packages/openai-base/src/adapters/responses-text.ts @@ -1,6 +1,6 @@ import { EventType, - assertOwnFileSource, + fileReferenceFor, isFileSource, normalizeSystemPrompts, unsupportedFileSourceError, @@ -61,15 +61,6 @@ export abstract class OpenAIBaseResponsesTextAdapter< readonly name: string protected client: OpenAI - /** - * Whether this provider's Responses endpoint accepts `file_id` references - * from its own Files API. Only OpenAI itself does — OpenAI-compatible - * subclasses (Grok, Bedrock, custom providers) have no Files API surface, - * so a `{ type: 'file' }` source is rejected instead of being sent as a - * `file_id` the provider can't resolve. - */ - protected readonly supportsFileIdInput: boolean = false - constructor(model: TModel, name: string, client: OpenAI) { super({}, model) this.name = name @@ -1815,13 +1806,12 @@ export abstract class OpenAIBaseResponsesTextAdapter< | { detail?: 'auto' | 'low' | 'high' } | undefined if (isFileSource(part.source)) { - if (!this.supportsFileIdInput) { + if (this.supportsFileSources !== true) { throw unsupportedFileSourceError(this.name) } - assertOwnFileSource(part.source, this.name) return { type: 'input_image', - file_id: part.source.value, + file_id: fileReferenceFor(part.source, this.name), detail: imageMetadata?.detail || 'auto', } } @@ -1849,13 +1839,12 @@ export abstract class OpenAIBaseResponsesTextAdapter< } case 'audio': { if (isFileSource(part.source)) { - if (!this.supportsFileIdInput) { + if (this.supportsFileSources !== true) { throw unsupportedFileSourceError(this.name) } - assertOwnFileSource(part.source, this.name) return { type: 'input_file', - file_id: part.source.value, + file_id: fileReferenceFor(part.source, this.name), } } if (part.source.type === 'url') { @@ -1883,13 +1872,12 @@ export abstract class OpenAIBaseResponsesTextAdapter< // This adapter doesn't map inline document bytes/URLs onto the // Responses `file_data`/`file_url` fields (yet) — only handles. if (isFileSource(part.source)) { - if (!this.supportsFileIdInput) { + if (this.supportsFileSources !== true) { throw unsupportedFileSourceError(this.name) } - assertOwnFileSource(part.source, this.name) return { type: 'input_file', - file_id: part.source.value, + file_id: fileReferenceFor(part.source, this.name), } } throw new Error(`Unsupported content part type: ${part.type}`) diff --git a/testing/e2e/src/routes/api.file-source-wire.ts b/testing/e2e/src/routes/api.file-source-wire.ts index 01f3e794a..2c25d3444 100644 --- a/testing/e2e/src/routes/api.file-source-wire.ts +++ b/testing/e2e/src/routes/api.file-source-wire.ts @@ -52,8 +52,7 @@ export const Route = createFileRoute('/api/file-source-wire')({ type: 'image', source: { type: 'file', - value: handleValue, - provider: handleProvider, + reference: { [handleProvider]: handleValue }, }, }, ], diff --git a/testing/e2e/tests/file-source-wire.spec.ts b/testing/e2e/tests/file-source-wire.spec.ts index 04d9bc245..bb4be84f2 100644 --- a/testing/e2e/tests/file-source-wire.spec.ts +++ b/testing/e2e/tests/file-source-wire.spec.ts @@ -44,7 +44,7 @@ test.describe('file content source — wire format', () => { expect(entries.length).toBeGreaterThan(0) }) - test('openai: a foreign (gemini) handle is rejected, not forwarded', async ({ + test('openai: a gemini-only reference is rejected — no openai entry in the record', async ({ request, testId, }) => { @@ -54,7 +54,7 @@ test.describe('file content source — wire format', () => { const { ok, error } = (await res.json()) as { ok: boolean; error?: string } expect(ok).toBe(false) expect(error).toMatch(/openai/) - expect(error).toMatch(/gemini/) + expect(error).toMatch(/found: gemini/) }) test('anthropic: an own-provider handle completes the round-trip (file_id block covered by unit test)', async ({ @@ -83,7 +83,7 @@ test.describe('file content source — wire format', () => { expect(ok).toBe(true) }) - test('gemini: a foreign (openai) handle is rejected before any request', async ({ + test('gemini: an openai-only reference is rejected before any request', async ({ request, testId, }) => { @@ -93,5 +93,6 @@ test.describe('file content source — wire format', () => { const { ok, error } = (await res.json()) as { ok: boolean; error?: string } expect(ok).toBe(false) expect(error).toMatch(/gemini/) + expect(error).toMatch(/found: openai/) }) })