Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/fix-ai-chat-abort-cancel.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@cloudflare/ai-chat": patch
---

Fix abort/cancel support for streaming responses. The framework now properly cancels the reader loop when the abort signal fires and sends a done signal to the client. Added a warning log when cancellation arrives but the stream has not closed (indicating the user forgot to pass `abortSignal` to their LLM call). Also fixed vitest project configs to scope test file discovery and prevent e2e/react tests from being picked up by the wrong runner.
33 changes: 23 additions & 10 deletions examples/ai-chat/src/client.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
} from "@cloudflare/agents-ui";
import {
PaperPlaneRightIcon,
StopIcon,
TrashIcon,
CheckCircleIcon,
XCircleIcon,
Expand Down Expand Up @@ -54,6 +55,7 @@ function Chat() {
sendMessage,
clearHistory,
addToolApprovalResponse,
stop,
status
} = useAgentChat({
agent,
Expand Down Expand Up @@ -333,16 +335,27 @@ function Chat() {
rows={2}
className="flex-1 !ring-0 focus:!ring-0 !shadow-none !bg-transparent !outline-none"
/>
<Button
type="submit"
variant="primary"
shape="square"
aria-label="Send message"
disabled={!input.trim() || !isConnected || isStreaming}
icon={<PaperPlaneRightIcon size={18} />}
loading={isStreaming}
className="mb-0.5"
/>
{isStreaming ? (
<Button
type="button"
variant="secondary"
shape="square"
aria-label="Stop streaming"
onClick={stop}
icon={<StopIcon size={18} weight="fill" />}
className="mb-0.5"
/>
) : (
<Button
type="submit"
variant="primary"
shape="square"
aria-label="Send message"
disabled={!input.trim() || !isConnected}
icon={<PaperPlaneRightIcon size={18} />}
className="mb-0.5"
/>
)}
</div>
</form>
<div className="flex justify-center pb-3">
Expand Down
5 changes: 3 additions & 2 deletions examples/ai-chat/src/server.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { createWorkersAI } from "workers-ai-provider";
import { routeAgentRequest } from "agents";
import { AIChatAgent } from "@cloudflare/ai-chat";
import { AIChatAgent, type OnChatMessageOptions } from "@cloudflare/ai-chat";
import {
streamText,
convertToModelMessages,
Expand All @@ -23,10 +23,11 @@ export class ChatAgent extends AIChatAgent {
// Keep the last 200 messages in SQLite storage
maxPersistedMessages = 200;

async onChatMessage() {
async onChatMessage(_onFinish: unknown, options?: OnChatMessageOptions) {
const workersai = createWorkersAI({ binding: this.env.AI });

const result = streamText({
abortSignal: options?.abortSignal,
model: workersai("@cf/zai-org/glm-4.7-flash"),
system:
"You are a helpful assistant. You can check the weather, get the user's timezone, " +
Expand Down
106 changes: 100 additions & 6 deletions packages/ai-chat/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1669,11 +1669,35 @@ export class AIChatAgent<
reader: ReadableStreamDefaultReader<Uint8Array>,
message: ChatMessage,
streamCompleted: { value: boolean },
continuation = false
continuation = false,
abortSignal?: AbortSignal
) {
streamCompleted.value = false;

// Cancel the reader when the abort signal fires (e.g. client pressed stop).
// This ensures we stop broadcasting chunks even if the underlying stream
// hasn't been connected to the abort signal (e.g. user forgot to pass it
// to streamText).
if (abortSignal && !abortSignal.aborted) {
abortSignal.addEventListener(
"abort",
() => {
reader.cancel().catch(() => {});
},
{ once: true }
);
}

while (true) {
const { done, value } = await reader.read();
if (abortSignal?.aborted) break;
let readResult: ReadableStreamReadResult<Uint8Array>;
try {
readResult = await reader.read();
} catch {
// reader.read() throws after cancel() — treat as abort
break;
}
const { done, value } = readResult;
if (done) {
this._completeStream(streamId);
streamCompleted.value = true;
Expand Down Expand Up @@ -1861,6 +1885,24 @@ export class AIChatAgent<
}
}
}

// If we exited due to abort, send a done signal so clients know the stream ended
if (!streamCompleted.value) {
console.warn(
"[AIChatAgent] Stream was still active when cancel was received. " +
"Pass options.abortSignal to streamText() in your onChatMessage() " +
"to cancel the upstream LLM call and avoid wasted work."
);
this._completeStream(streamId);
streamCompleted.value = true;
this._broadcastChatMessage({
body: "",
done: true,
id,
type: MessageType.CF_AGENT_USE_CHAT_RESPONSE,
...(continuation && { continuation: true })
});
}
}

// Handle plain text responses (e.g., from generateText)
Expand All @@ -1870,7 +1912,8 @@ export class AIChatAgent<
reader: ReadableStreamDefaultReader<Uint8Array>,
message: ChatMessage,
streamCompleted: { value: boolean },
continuation = false
continuation = false,
abortSignal?: AbortSignal
) {
// if not AI SDK SSE format, we need to inject text-start and text-end events ourselves
this._broadcastTextEvent(
Expand All @@ -1884,8 +1927,27 @@ export class AIChatAgent<
const textPart: TextUIPart = { type: "text", text: "", state: "streaming" };
message.parts.push(textPart);

// Cancel the reader when the abort signal fires
if (abortSignal && !abortSignal.aborted) {
abortSignal.addEventListener(
"abort",
() => {
reader.cancel().catch(() => {});
},
{ once: true }
);
}

while (true) {
const { done, value } = await reader.read();
if (abortSignal?.aborted) break;
let readResult: ReadableStreamReadResult<Uint8Array>;
try {
readResult = await reader.read();
} catch {
// reader.read() throws after cancel() — treat as abort
break;
}
const { done, value } = readResult;
if (done) {
textPart.state = "done";

Expand Down Expand Up @@ -1921,6 +1983,30 @@ export class AIChatAgent<
);
}
}

// If we exited due to abort, send a done signal so clients know the stream ended
if (!streamCompleted.value) {
console.warn(
"[AIChatAgent] Stream was still active when cancel was received. " +
"Pass options.abortSignal to streamText() in your onChatMessage() " +
"to cancel the upstream LLM call and avoid wasted work."
);
textPart.state = "done";
this._broadcastTextEvent(
streamId,
{ type: "text-end", id },
continuation
);
this._completeStream(streamId);
streamCompleted.value = true;
this._broadcastChatMessage({
body: "",
done: true,
id,
type: MessageType.CF_AGENT_USE_CHAT_RESPONSE,
...(continuation && { continuation: true })
});
}
}

/**
Expand Down Expand Up @@ -1964,6 +2050,12 @@ export class AIChatAgent<
options: { continuation?: boolean; chatMessageId?: string } = {}
) {
const { continuation = false, chatMessageId } = options;
// Look up the abort signal for this request so we can cancel the reader
// loop if the client sends a cancel message. This is a safety net —
// users should also pass abortSignal to streamText for proper cancellation.
const abortSignal = chatMessageId
? this._chatMessageAbortControllers.get(chatMessageId)?.signal
: undefined;

return this._tryCatchChat(async () => {
if (!response.body) {
Expand Down Expand Up @@ -2015,7 +2107,8 @@ export class AIChatAgent<
reader,
message,
streamCompleted,
continuation
continuation,
abortSignal
);
} else {
await this._sendPlaintextReply(
Expand All @@ -2024,7 +2117,8 @@ export class AIChatAgent<
reader,
message,
streamCompleted,
continuation
continuation,
abortSignal
);
}
} catch (error) {
Expand Down
1 change: 1 addition & 0 deletions packages/ai-chat/src/react-tests/vitest.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { defineConfig } from "vitest/config";
export default defineConfig({
test: {
name: "react",
include: ["**/*.test.{ts,tsx}"],
browser: {
enabled: true,
instances: [
Expand Down
Loading
Loading