From a5a9fe679303f86e2d68fc3b718aa556ccd33b7d Mon Sep 17 00:00:00 2001 From: T3 Code PR Stack <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:50:54 +0200 Subject: [PATCH] fix(discord-bot): survive guest T3 restarts without stuck project lookups Boot used a single connect attempt and could mark the session ready before the shell snapshot arrived, so mentions after a restart looked like missing projects and stayed broken until a manual bot restart. Retry connect until shell-ready, tear down partial sessions, force reconnect on transport errors, and surface a transient "still connecting" message instead of a false alias/path failure. --- .../discord-bot/src/features/MentionRouter.ts | 31 ++++- apps/discord-bot/src/main.ts | 22 +-- apps/discord-bot/src/t3/T3Session.test.ts | 24 +++- apps/discord-bot/src/t3/T3Session.ts | 130 ++++++++++++++++-- docs/integrations/discord-bot.md | 9 ++ 5 files changed, 188 insertions(+), 28 deletions(-) diff --git a/apps/discord-bot/src/features/MentionRouter.ts b/apps/discord-bot/src/features/MentionRouter.ts index a33cd424b0e..f04f056e742 100644 --- a/apps/discord-bot/src/features/MentionRouter.ts +++ b/apps/discord-bot/src/features/MentionRouter.ts @@ -76,7 +76,7 @@ import { IdentityMapStore } from "../identityMap.ts"; import { ProjectAliasStore } from "../projectAliases.ts"; import { type ThreadLink, ThreadLinkStore } from "../store/ThreadLinkStore.ts"; import { newMessageId } from "../t3/ids.ts"; -import { T3Session, T3SessionError } from "../t3/T3Session.ts"; +import { T3_STILL_CONNECTING_MESSAGE, T3Session, T3SessionError } from "../t3/T3Session.ts"; import { formatAlertCause } from "./Alerts.ts"; import { ensureChannelInfoPin } from "./ChannelInfoPin.ts"; import { makeDiscordThreadTurnCoordinator } from "./DiscordThreadTurnCoordinator.ts"; @@ -596,6 +596,10 @@ const make = (botConfig: DiscordBotConfig) => } const project = yield* t3.findProjectByWorkspaceRoot(alias.workspaceRoot); if (project === null) { + const ready = yield* t3.isReady(); + if (!ready) { + return yield* Effect.fail(T3_STILL_CONNECTING_MESSAGE); + } return yield* Effect.fail( `No T3 project registered at ${alias.workspaceRoot} (alias '${shortName}'). Add the project in T3 first.` as const, ); @@ -646,6 +650,14 @@ const make = (botConfig: DiscordBotConfig) => } const project = yield* t3.findProjectByWorkspaceRoot(alias.workspaceRoot); if (project === null) { + const ready = yield* t3.isReady(); + if (!ready) { + return { + kind: "t3-connecting" as const, + shortName, + workspaceRoot: alias.workspaceRoot, + }; + } return { kind: "no-project" as const, shortName, @@ -1257,12 +1269,16 @@ const make = (botConfig: DiscordBotConfig) => }); }); - const reportError = (channelId: string, error: unknown) => - rest - .createMessage(channelId, { - content: `Could not start T3 turn: ${error instanceof Error ? error.message : String(error)}`, - }) + const reportError = (channelId: string, error: unknown) => { + const raw = error instanceof Error ? error.message : String(error); + const content = + raw === T3_STILL_CONNECTING_MESSAGE || /not connected|still connecting/i.test(raw) + ? T3_STILL_CONNECTING_MESSAGE + : `Could not start T3 turn: ${raw}`; + return rest + .createMessage(channelId, { content }) .pipe(Effect.catchCause(Effect.logError), Effect.asVoid); + }; /** * dfx InteractionsRegistry runs Ix handlers with a Discord-only context — app @@ -2981,6 +2997,9 @@ const make = (botConfig: DiscordBotConfig) => { ephemeral: true }, ); } + if (binding.kind === "t3-connecting") { + return slashReply(T3_STILL_CONNECTING_MESSAGE, { ephemeral: true }); + } if (binding.kind === "no-project") { return slashReply( `Alias \`${binding.shortName}\` → \`${binding.workspaceRoot}\`, but no T3 project is registered at that path.`, diff --git a/apps/discord-bot/src/main.ts b/apps/discord-bot/src/main.ts index a71d3d224c9..28723960635 100644 --- a/apps/discord-bot/src/main.ts +++ b/apps/discord-bot/src/main.ts @@ -86,20 +86,18 @@ const program = Effect.gen(function* () { }); // Force acquisition of MentionRouter + Discord gateway (must not be pruned). + // Discord can become READY before guest T3 is listening — that is expected on + // restart. Mentions that land early get a transient "still connecting" reply. const running = yield* DiscordBotRunning; yield* Effect.logInfo("Discord gateway active", { botUserId: running.botUserId }); const t3 = yield* T3Session; - yield* t3.connect(); - const aliasStore = yield* ProjectAliasStore; - const identityStore = yield* IdentityMapStore; - yield* Effect.logInfo( - `Connected to T3; bot project aliases=${aliasStore.list().length}; identity map entries=${identityStore.list().length}`, - ); // Capture ambient services so T3 auto-reconnect can rehydrate Discord bridges. - // provideContext erases R at runtime; cast so Effect.runPromise accepts the program. - // (setOnReconnected is a Promise callback outside the Effect runtime, so runPromise is intentional.) + // Register before connectUntilReady so a drop immediately after first success + // still rehydrates. provideContext erases R at runtime; cast so Effect.runPromise + // accepts the program. (setOnReconnected is a Promise callback outside the Effect + // runtime, so runPromise is intentional.) const services = yield* Effect.context(); t3.setOnReconnected(() => // @effect-diagnostics-next-line runEffectInsideEffect:off @@ -116,6 +114,14 @@ const program = Effect.gen(function* () { ), ); + // Retry forever until shell is ready — do not exit the process when T3 is late. + yield* t3.connectUntilReady(); + const aliasStore = yield* ProjectAliasStore; + const identityStore = yield* IdentityMapStore; + yield* Effect.logInfo( + `Connected to T3; bot project aliases=${aliasStore.list().length}; identity map entries=${identityStore.list().length}`, + ); + // Restore running/pending bridges + catch-up finalize for open stream tips. // Without this, mid-turn Discord threads go silent until a human @mentions again. yield* rehydrateBridges("boot").pipe( diff --git a/apps/discord-bot/src/t3/T3Session.test.ts b/apps/discord-bot/src/t3/T3Session.test.ts index 41b91769908..31d23f5ca90 100644 --- a/apps/discord-bot/src/t3/T3Session.test.ts +++ b/apps/discord-bot/src/t3/T3Session.test.ts @@ -1,7 +1,29 @@ import { ProviderInstanceId } from "@t3tools/contracts"; import { describe, expect, it } from "vite-plus/test"; -import { shouldPersistThreadModelSelectionForNextTurn } from "./T3Session.ts"; +import { + isT3TransportError, + shouldPersistThreadModelSelectionForNextTurn, + T3_STILL_CONNECTING_MESSAGE, +} from "./T3Session.ts"; + +describe("isT3TransportError", () => { + it("matches SocketCloseError and ConnectionTransientError", () => { + expect(isT3TransportError(new Error("SocketCloseError: 1005"))).toBe(true); + expect(isT3TransportError(new Error("ConnectionTransientError: closed"))).toBe(true); + expect(isT3TransportError(new Error("SocketError: reset"))).toBe(true); + }); + + it("matches common network errno strings", () => { + expect(isT3TransportError(new Error("connect ECONNREFUSED 127.0.0.1:3773"))).toBe(true); + expect(isT3TransportError(new Error("read ECONNRESET"))).toBe(true); + }); + + it("does not match ordinary application errors", () => { + expect(isT3TransportError(new Error("No T3 project registered at /tmp/x"))).toBe(false); + expect(isT3TransportError(new Error(T3_STILL_CONNECTING_MESSAGE))).toBe(false); + }); +}); describe("shouldPersistThreadModelSelectionForNextTurn", () => { it("returns false when no explicit model selection is provided", () => { diff --git a/apps/discord-bot/src/t3/T3Session.ts b/apps/discord-bot/src/t3/T3Session.ts index cc340f18b7d..eb6b5253540 100644 --- a/apps/discord-bot/src/t3/T3Session.ts +++ b/apps/discord-bot/src/t3/T3Session.ts @@ -52,6 +52,7 @@ import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as ManagedRuntime from "effect/ManagedRuntime"; +import * as Result from "effect/Result"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as Socket from "effect/unstable/socket/Socket"; @@ -59,7 +60,6 @@ import * as Socket from "effect/unstable/socket/Socket"; import type { DiscordBotConfig } from "../config.ts"; import { preferredModelSelection } from "../config.ts"; import { BrowserAutomationHost } from "../browser/BrowserAutomationHost.ts"; -import { formatAlertCause } from "../features/Alerts.ts"; import { formatThreadTitle } from "../presentation/messages.ts"; import { normalizeWorkspacePath } from "../presentation/mentions.ts"; import { followOrchestrationThread } from "./DiscordThreadFollower.ts"; @@ -84,6 +84,23 @@ function messageFromCause(cause: unknown): string { return String(cause); } +/** + * Detect dead/transient socket failures that should force a reconnect when + * `RpcSession.closed` did not fire (or fired too late). Used for soft recovery + * on dispatch errors — we do **not** auto-retry the command (double-start risk). + */ +export function isT3TransportError(cause: unknown): boolean { + const msg = messageFromCause(cause); + if (/SocketCloseError|ConnectionTransientError|SocketError/i.test(msg)) return true; + if (/ECONNREFUSED|ECONNRESET|ENOTFOUND|EPIPE|ETIMEDOUT/i.test(msg)) return true; + if (/websocket/i.test(msg) && /close|closed|reset|refused|not connected/i.test(msg)) return true; + return false; +} + +/** User-facing copy when the bot is online but T3 is still (re)connecting. */ +export const T3_STILL_CONNECTING_MESSAGE = + "T3 is still connecting after a server restart (or first boot). Try again in a few seconds."; + export function shouldPersistThreadModelSelectionForNextTurn(input: { readonly currentModelSelection?: ModelSelection; readonly nextModelSelection?: ModelSelection; @@ -108,11 +125,24 @@ export class T3SessionError extends Error { export interface T3SessionService { readonly connect: () => Effect.Effect; + /** + * Keep retrying {@link connect} with the same backoff as mid-life reconnect + * until the shell snapshot is live. Used at boot so the process does not exit + * when Discord comes up before T3 (guest restart race). + */ + readonly connectUntilReady: () => Effect.Effect; /** * Register a callback invoked after a successful automatic reconnect * (socket drop → reconnectLoop). Used to rehydrate Discord bridges. */ readonly setOnReconnected: (handler: (() => Promise) | null) => void; + /** + * True when a live RpcSession exists (may still be waiting for shell on a + * race; prefer {@link isReady} before project lookups). + */ + readonly isConnected: () => Effect.Effect; + /** True when connected and the orchestration shell snapshot has arrived. */ + readonly isReady: () => Effect.Effect; readonly shell: () => Effect.Effect; readonly serverConfig: () => Effect.Effect; readonly findProjectByWorkspaceRoot: ( @@ -241,6 +271,13 @@ export class T3Session extends Context.Service()( */ const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const; +/** How long to wait for the first shell snapshot after the WS is up (ms poll × attempts). */ +const SHELL_SNAPSHOT_POLL_MS = 100; +const SHELL_SNAPSHOT_MAX_ATTEMPTS = 150; // 15s — guest T3 can lag Discord on restart + +/** Brief wait for shell when a mention races reconnect completion. */ +const PROJECT_LOOKUP_SHELL_WAIT_ATTEMPTS = 30; // 3s + export const makeT3Session = (botConfig: DiscordBotConfig) => Effect.sync(() => { const runtime = ManagedRuntime.make( @@ -265,7 +302,7 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => const threadFibers = new Map>(); const requireSession = (): RpcSession => { - if (session === null) throw new T3SessionError("T3 Code is not connected."); + if (session === null) throw new T3SessionError(T3_STILL_CONNECTING_MESSAGE); return session; }; @@ -309,6 +346,23 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => } }; + /** + * When the socket is dead but `closed` never fired (or dispatch saw the failure + * first), tear down and enter reconnectLoop. Does not re-run the failed command. + */ + const scheduleReconnectFromTransportError = (cause: unknown) => { + if (reconnecting) return; + void (async () => { + await runtime + .runPromise( + Effect.logWarning(`T3 transport error; forcing reconnect: ${messageFromCause(cause)}`), + ) + .catch(() => {}); + await teardown(); + await reconnectLoop(); + })(); + }; + /** * Reconnect with backoff, mirroring EnvironmentSupervisor's schedule * (packages/client-runtime/src/connection/supervisor.ts) which web/mobile already @@ -367,6 +421,9 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => connected.closed.pipe( Effect.catchCause((cause) => Effect.gen(function* () { + // Another fiber may already have torn down and started reconnect + // (e.g. soft recovery from a transport error on dispatch). + if (session !== connected) return; yield* Effect.logWarning(`T3 connection lost: ${messageFromCause(cause)}`); yield* Effect.promise(() => teardown()); yield* Effect.promise(() => reconnectLoop()); @@ -385,8 +442,12 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => runtime.runPromise( requireSession().client[ORCHESTRATION_WS_METHODS.dispatchCommand](command), ), - catch: (cause) => - new T3SessionError(`dispatch failed: ${messageFromCause(cause)}`, { cause }), + catch: (cause) => { + if (isT3TransportError(cause)) { + scheduleReconnectFromTransportError(cause); + } + return new T3SessionError(`dispatch failed: ${messageFromCause(cause)}`, { cause }); + }, }).pipe(Effect.asVoid); const claimBrowserHost = (threadId: ThreadId) => @@ -412,7 +473,11 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => Effect.tryPromise({ try: async () => { const normalizedBaseUrl = new URL(baseUrl).toString(); - if (session !== null) return; + // Half-open: session without shell must never short-circuit (stuck forever). + if (session !== null && shell !== null) return; + if (session !== null && shell === null) { + await teardown(); + } let environmentId = EnvironmentId.make(PRIMARY_LOCAL_ENVIRONMENT_ID); let socketUrl = localSocketUrl(normalizedBaseUrl); @@ -547,15 +612,27 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => } // `shell` is updated by the concurrently running subscription fiber. - // eslint-disable-next-line no-unmodified-loop-condition - for (let attempt = 0; attempt < 50 && shell === null; attempt += 1) { - await Effect.runPromise(Effect.sleep("100 millis")); + for (let attempt = 0; attempt < SHELL_SNAPSHOT_MAX_ATTEMPTS; attempt += 1) { + // eslint-disable-next-line no-unmodified-loop-condition -- shell is set by shellFiber + if (shell !== null) break; + await Effect.runPromise(Effect.sleep(Duration.millis(SHELL_SNAPSHOT_POLL_MS))); + } + if (shell === null) { + throw new T3SessionError( + "Connected to T3 but the orchestration shell snapshot did not arrive in time", + ); } } catch (cause) { - await runtime.runPromise(Scope.close(nextScope, Exit.void)); - throw new T3SessionError(`Could not connect to T3 Code: ${messageFromCause(cause)}`, { - cause, - }); + // Full teardown is required: a partial connect leaves session non-null and + // the next connectPrepared early-returns forever (stuck bot after restart). + await teardown(); + // nextScope may not have been published to `scope` yet (fail before assign). + await runtime.runPromise(Scope.close(nextScope, Exit.void)).catch(() => {}); + throw cause instanceof T3SessionError + ? cause + : new T3SessionError(`Could not connect to T3 Code: ${messageFromCause(cause)}`, { + cause, + }); } }, catch: (cause) => @@ -592,6 +669,24 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => yield* connectPrepared(botConfig.t3HttpBaseUrl); }); + /** + * Boot path: Discord may start before guest T3 is listening. Retry forever + * with the same schedule as mid-life reconnect instead of exiting the process. + */ + const connectUntilReady = () => + Effect.gen(function* () { + for (let attempt = 0; ; attempt += 1) { + const outcome = yield* Effect.result(connect()); + if (Result.isSuccess(outcome)) return; + const delay = + RECONNECT_DELAYS_MS[Math.min(attempt, RECONNECT_DELAYS_MS.length - 1)] ?? 16_000; + yield* Effect.logWarning( + `T3 boot connect attempt ${attempt + 1} failed: ${messageFromCause(outcome.failure)}; retrying in ${delay}ms`, + ); + yield* Effect.sleep(Duration.millis(delay)); + } + }); + const providersForSelection = (): ReadonlyArray => serverConfig?.providers ?? []; @@ -629,13 +724,22 @@ export const makeT3Session = (botConfig: DiscordBotConfig) => return T3Session.of({ connect, + connectUntilReady, setOnReconnected: (handler) => { onReconnected = handler; }, + isConnected: () => Effect.sync(() => session !== null), + isReady: () => Effect.sync(() => session !== null && shell !== null), shell: () => Effect.succeed(shell), serverConfig: () => Effect.succeed(serverConfig), findProjectByWorkspaceRoot: (workspaceRoot) => - Effect.sync(() => { + Effect.gen(function* () { + // Mentions can land while shell is still filling after reconnect. + for (let attempt = 0; attempt < PROJECT_LOOKUP_SHELL_WAIT_ATTEMPTS; attempt += 1) { + // shell/session are mutated by connect/teardown/shellFiber, not this loop body. + if (shell !== null || session === null) break; + yield* Effect.sleep(Duration.millis(SHELL_SNAPSHOT_POLL_MS)); + } if (shell === null) return null; const target = normalizeWorkspacePath(workspaceRoot); return ( diff --git a/docs/integrations/discord-bot.md b/docs/integrations/discord-bot.md index 032a2802656..1331513d3d9 100644 --- a/docs/integrations/discord-bot.md +++ b/docs/integrations/discord-bot.md @@ -241,6 +241,15 @@ finalize only deletes stored stream tip ids — it never scans the channel for o markers. Design notes: [`apps/discord-bot/DESIGN-thread-restore.md`](../../apps/discord-bot/DESIGN-thread-restore.md). +**Guest restart / Discord-before-T3 race** (implemented): the Discord gateway can come READY +before the guest T3 process is listening. Boot uses `connectUntilReady` (same backoff as +mid-life reconnect) and does **not** exit the process when T3 is late. Connect is only +considered successful once the orchestration **shell snapshot** has arrived (so project +lookups are not empty). Mid-life drops still tear down + reconnect; transport errors on +dispatch also force reconnect when `RpcSession.closed` is slow/missed. Mentions that land +while T3 is reconnecting get a transient “still connecting… try again” reply instead of a +false “no T3 project registered at this path” error. + ## Architecture notes - Project shortName mapping lives **only on the Discord bot** — not in T3 server config/DB.