Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions apps/discord-bot/src/features/MentionRouter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ import { IdentityMapStore } from "../identityMap.ts";
import { ProjectAliasStore } from "../projectAliases.ts";
import { type ThreadLink, ThreadLinkStore } from "../store/ThreadLinkStore.ts";
import { newMessageId } from "../t3/ids.ts";
import { T3Session, T3SessionError } from "../t3/T3Session.ts";
import { T3_STILL_CONNECTING_MESSAGE, T3Session, T3SessionError } from "../t3/T3Session.ts";
import { formatAlertCause } from "./Alerts.ts";
import { ensureChannelInfoPin } from "./ChannelInfoPin.ts";
import { makeDiscordThreadTurnCoordinator } from "./DiscordThreadTurnCoordinator.ts";
Expand Down Expand Up @@ -596,6 +596,10 @@ const make = (botConfig: DiscordBotConfig) =>
}
const project = yield* t3.findProjectByWorkspaceRoot(alias.workspaceRoot);
if (project === null) {
const ready = yield* t3.isReady();
if (!ready) {
return yield* Effect.fail(T3_STILL_CONNECTING_MESSAGE);
}
return yield* Effect.fail(
`No T3 project registered at ${alias.workspaceRoot} (alias '${shortName}'). Add the project in T3 first.` as const,
);
Expand Down Expand Up @@ -646,6 +650,14 @@ const make = (botConfig: DiscordBotConfig) =>
}
const project = yield* t3.findProjectByWorkspaceRoot(alias.workspaceRoot);
if (project === null) {
const ready = yield* t3.isReady();
if (!ready) {
return {
kind: "t3-connecting" as const,
shortName,
workspaceRoot: alias.workspaceRoot,
};
}
return {
kind: "no-project" as const,
shortName,
Expand Down Expand Up @@ -1257,12 +1269,16 @@ const make = (botConfig: DiscordBotConfig) =>
});
});

const reportError = (channelId: string, error: unknown) =>
rest
.createMessage(channelId, {
content: `Could not start T3 turn: ${error instanceof Error ? error.message : String(error)}`,
})
const reportError = (channelId: string, error: unknown) => {
const raw = error instanceof Error ? error.message : String(error);
const content =
raw === T3_STILL_CONNECTING_MESSAGE || /not connected|still connecting/i.test(raw)
? T3_STILL_CONNECTING_MESSAGE
: `Could not start T3 turn: ${raw}`;
return rest
.createMessage(channelId, { content })
.pipe(Effect.catchCause(Effect.logError), Effect.asVoid);
};

/**
* dfx InteractionsRegistry runs Ix handlers with a Discord-only context — app
Expand Down Expand Up @@ -2981,6 +2997,9 @@ const make = (botConfig: DiscordBotConfig) =>
{ ephemeral: true },
);
}
if (binding.kind === "t3-connecting") {
return slashReply(T3_STILL_CONNECTING_MESSAGE, { ephemeral: true });
}
if (binding.kind === "no-project") {
return slashReply(
`Alias \`${binding.shortName}\` → \`${binding.workspaceRoot}\`, but no T3 project is registered at that path.`,
Expand Down
22 changes: 14 additions & 8 deletions apps/discord-bot/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,20 +86,18 @@ const program = Effect.gen(function* () {
});

// Force acquisition of MentionRouter + Discord gateway (must not be pruned).
// Discord can become READY before guest T3 is listening — that is expected on
// restart. Mentions that land early get a transient "still connecting" reply.
const running = yield* DiscordBotRunning;
yield* Effect.logInfo("Discord gateway active", { botUserId: running.botUserId });

const t3 = yield* T3Session;
yield* t3.connect();
const aliasStore = yield* ProjectAliasStore;
const identityStore = yield* IdentityMapStore;
yield* Effect.logInfo(
`Connected to T3; bot project aliases=${aliasStore.list().length}; identity map entries=${identityStore.list().length}`,
);

// Capture ambient services so T3 auto-reconnect can rehydrate Discord bridges.
// provideContext erases R at runtime; cast so Effect.runPromise accepts the program.
// (setOnReconnected is a Promise callback outside the Effect runtime, so runPromise is intentional.)
// Register before connectUntilReady so a drop immediately after first success
// still rehydrates. provideContext erases R at runtime; cast so Effect.runPromise
// accepts the program. (setOnReconnected is a Promise callback outside the Effect
// runtime, so runPromise is intentional.)
const services = yield* Effect.context();
t3.setOnReconnected(() =>
// @effect-diagnostics-next-line runEffectInsideEffect:off
Expand All @@ -116,6 +114,14 @@ const program = Effect.gen(function* () {
),
);

// Retry forever until shell is ready — do not exit the process when T3 is late.
yield* t3.connectUntilReady();
const aliasStore = yield* ProjectAliasStore;
const identityStore = yield* IdentityMapStore;
yield* Effect.logInfo(
`Connected to T3; bot project aliases=${aliasStore.list().length}; identity map entries=${identityStore.list().length}`,
);

// Restore running/pending bridges + catch-up finalize for open stream tips.
// Without this, mid-turn Discord threads go silent until a human @mentions again.
yield* rehydrateBridges("boot").pipe(
Expand Down
24 changes: 23 additions & 1 deletion apps/discord-bot/src/t3/T3Session.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,29 @@
import { ProviderInstanceId } from "@t3tools/contracts";
import { describe, expect, it } from "vite-plus/test";

import { shouldPersistThreadModelSelectionForNextTurn } from "./T3Session.ts";
import {
isT3TransportError,
shouldPersistThreadModelSelectionForNextTurn,
T3_STILL_CONNECTING_MESSAGE,
} from "./T3Session.ts";

describe("isT3TransportError", () => {
it("matches SocketCloseError and ConnectionTransientError", () => {
expect(isT3TransportError(new Error("SocketCloseError: 1005"))).toBe(true);
expect(isT3TransportError(new Error("ConnectionTransientError: closed"))).toBe(true);
expect(isT3TransportError(new Error("SocketError: reset"))).toBe(true);
});

it("matches common network errno strings", () => {
expect(isT3TransportError(new Error("connect ECONNREFUSED 127.0.0.1:3773"))).toBe(true);
expect(isT3TransportError(new Error("read ECONNRESET"))).toBe(true);
});

it("does not match ordinary application errors", () => {
expect(isT3TransportError(new Error("No T3 project registered at /tmp/x"))).toBe(false);
expect(isT3TransportError(new Error(T3_STILL_CONNECTING_MESSAGE))).toBe(false);
});
});

describe("shouldPersistThreadModelSelectionForNextTurn", () => {
it("returns false when no explicit model selection is provided", () => {
Expand Down
130 changes: 117 additions & 13 deletions apps/discord-bot/src/t3/T3Session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,14 +52,14 @@ import * as Exit from "effect/Exit";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as ManagedRuntime from "effect/ManagedRuntime";
import * as Result from "effect/Result";
import * as Scope from "effect/Scope";
import * as Stream from "effect/Stream";
import * as Socket from "effect/unstable/socket/Socket";

import type { DiscordBotConfig } from "../config.ts";
import { preferredModelSelection } from "../config.ts";
import { BrowserAutomationHost } from "../browser/BrowserAutomationHost.ts";
import { formatAlertCause } from "../features/Alerts.ts";
import { formatThreadTitle } from "../presentation/messages.ts";
import { normalizeWorkspacePath } from "../presentation/mentions.ts";
import { followOrchestrationThread } from "./DiscordThreadFollower.ts";
Expand All @@ -84,6 +84,23 @@ function messageFromCause(cause: unknown): string {
return String(cause);
}

/**
* Detect dead/transient socket failures that should force a reconnect when
* `RpcSession.closed` did not fire (or fired too late). Used for soft recovery
* on dispatch errors — we do **not** auto-retry the command (double-start risk).
*/
export function isT3TransportError(cause: unknown): boolean {
const msg = messageFromCause(cause);
if (/SocketCloseError|ConnectionTransientError|SocketError/i.test(msg)) return true;
if (/ECONNREFUSED|ECONNRESET|ENOTFOUND|EPIPE|ETIMEDOUT/i.test(msg)) return true;
if (/websocket/i.test(msg) && /close|closed|reset|refused|not connected/i.test(msg)) return true;
return false;
}

/** User-facing copy when the bot is online but T3 is still (re)connecting. */
export const T3_STILL_CONNECTING_MESSAGE =
"T3 is still connecting after a server restart (or first boot). Try again in a few seconds.";

export function shouldPersistThreadModelSelectionForNextTurn(input: {
readonly currentModelSelection?: ModelSelection;
readonly nextModelSelection?: ModelSelection;
Expand All @@ -108,11 +125,24 @@ export class T3SessionError extends Error {

export interface T3SessionService {
readonly connect: () => Effect.Effect<void, T3SessionError>;
/**
* Keep retrying {@link connect} with the same backoff as mid-life reconnect
* until the shell snapshot is live. Used at boot so the process does not exit
* when Discord comes up before T3 (guest restart race).
*/
readonly connectUntilReady: () => Effect.Effect<void>;
/**
* Register a callback invoked after a successful automatic reconnect
* (socket drop → reconnectLoop). Used to rehydrate Discord bridges.
*/
readonly setOnReconnected: (handler: (() => Promise<void>) | null) => void;
/**
* True when a live RpcSession exists (may still be waiting for shell on a
* race; prefer {@link isReady} before project lookups).
*/
readonly isConnected: () => Effect.Effect<boolean>;
/** True when connected and the orchestration shell snapshot has arrived. */
readonly isReady: () => Effect.Effect<boolean>;
readonly shell: () => Effect.Effect<OrchestrationShellSnapshot | null>;
readonly serverConfig: () => Effect.Effect<ServerConfig | null>;
readonly findProjectByWorkspaceRoot: (
Expand Down Expand Up @@ -241,6 +271,13 @@ export class T3Session extends Context.Service<T3Session, T3SessionService>()(
*/
const RECONNECT_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const;

/** How long to wait for the first shell snapshot after the WS is up (ms poll × attempts). */
const SHELL_SNAPSHOT_POLL_MS = 100;
const SHELL_SNAPSHOT_MAX_ATTEMPTS = 150; // 15s — guest T3 can lag Discord on restart

/** Brief wait for shell when a mention races reconnect completion. */
const PROJECT_LOOKUP_SHELL_WAIT_ATTEMPTS = 30; // 3s

export const makeT3Session = (botConfig: DiscordBotConfig) =>
Effect.sync(() => {
const runtime = ManagedRuntime.make(
Expand All @@ -265,7 +302,7 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
const threadFibers = new Map<string, Fiber.Fiber<void, unknown>>();

const requireSession = (): RpcSession => {
if (session === null) throw new T3SessionError("T3 Code is not connected.");
if (session === null) throw new T3SessionError(T3_STILL_CONNECTING_MESSAGE);
return session;
};

Expand Down Expand Up @@ -309,6 +346,23 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
}
};

/**
* When the socket is dead but `closed` never fired (or dispatch saw the failure
* first), tear down and enter reconnectLoop. Does not re-run the failed command.
*/
const scheduleReconnectFromTransportError = (cause: unknown) => {
if (reconnecting) return;
void (async () => {
await runtime
.runPromise(
Effect.logWarning(`T3 transport error; forcing reconnect: ${messageFromCause(cause)}`),
)
.catch(() => {});
await teardown();
await reconnectLoop();
})();
};

/**
* Reconnect with backoff, mirroring EnvironmentSupervisor's schedule
* (packages/client-runtime/src/connection/supervisor.ts) which web/mobile already
Expand Down Expand Up @@ -367,6 +421,9 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
connected.closed.pipe(
Effect.catchCause((cause) =>
Effect.gen(function* () {
// Another fiber may already have torn down and started reconnect
// (e.g. soft recovery from a transport error on dispatch).
if (session !== connected) return;
yield* Effect.logWarning(`T3 connection lost: ${messageFromCause(cause)}`);
yield* Effect.promise(() => teardown());
yield* Effect.promise(() => reconnectLoop());
Expand All @@ -385,8 +442,12 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
runtime.runPromise(
requireSession().client[ORCHESTRATION_WS_METHODS.dispatchCommand](command),
),
catch: (cause) =>
new T3SessionError(`dispatch failed: ${messageFromCause(cause)}`, { cause }),
catch: (cause) => {
if (isT3TransportError(cause)) {
scheduleReconnectFromTransportError(cause);
}
return new T3SessionError(`dispatch failed: ${messageFromCause(cause)}`, { cause });
},
}).pipe(Effect.asVoid);

const claimBrowserHost = (threadId: ThreadId) =>
Expand All @@ -412,7 +473,11 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
Effect.tryPromise({
try: async () => {
const normalizedBaseUrl = new URL(baseUrl).toString();
if (session !== null) return;
// Half-open: session without shell must never short-circuit (stuck forever).
if (session !== null && shell !== null) return;
if (session !== null && shell === null) {
await teardown();
}

let environmentId = EnvironmentId.make(PRIMARY_LOCAL_ENVIRONMENT_ID);
let socketUrl = localSocketUrl(normalizedBaseUrl);
Expand Down Expand Up @@ -547,15 +612,27 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
}

// `shell` is updated by the concurrently running subscription fiber.
// eslint-disable-next-line no-unmodified-loop-condition
for (let attempt = 0; attempt < 50 && shell === null; attempt += 1) {
await Effect.runPromise(Effect.sleep("100 millis"));
for (let attempt = 0; attempt < SHELL_SNAPSHOT_MAX_ATTEMPTS; attempt += 1) {
// eslint-disable-next-line no-unmodified-loop-condition -- shell is set by shellFiber
if (shell !== null) break;
await Effect.runPromise(Effect.sleep(Duration.millis(SHELL_SNAPSHOT_POLL_MS)));
}
if (shell === null) {
throw new T3SessionError(
"Connected to T3 but the orchestration shell snapshot did not arrive in time",
);
}
} catch (cause) {
await runtime.runPromise(Scope.close(nextScope, Exit.void));
throw new T3SessionError(`Could not connect to T3 Code: ${messageFromCause(cause)}`, {
cause,
});
// Full teardown is required: a partial connect leaves session non-null and
// the next connectPrepared early-returns forever (stuck bot after restart).
await teardown();
// nextScope may not have been published to `scope` yet (fail before assign).
await runtime.runPromise(Scope.close(nextScope, Exit.void)).catch(() => {});
throw cause instanceof T3SessionError
? cause
: new T3SessionError(`Could not connect to T3 Code: ${messageFromCause(cause)}`, {
cause,
});
}
},
catch: (cause) =>
Expand Down Expand Up @@ -592,6 +669,24 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>
yield* connectPrepared(botConfig.t3HttpBaseUrl);
});

/**
* Boot path: Discord may start before guest T3 is listening. Retry forever
* with the same schedule as mid-life reconnect instead of exiting the process.
*/
const connectUntilReady = () =>
Effect.gen(function* () {
for (let attempt = 0; ; attempt += 1) {
const outcome = yield* Effect.result(connect());
if (Result.isSuccess(outcome)) return;
const delay =
RECONNECT_DELAYS_MS[Math.min(attempt, RECONNECT_DELAYS_MS.length - 1)] ?? 16_000;
yield* Effect.logWarning(
`T3 boot connect attempt ${attempt + 1} failed: ${messageFromCause(outcome.failure)}; retrying in ${delay}ms`,
);
yield* Effect.sleep(Duration.millis(delay));
}
});

const providersForSelection = (): ReadonlyArray<ServerProvider> =>
serverConfig?.providers ?? [];

Expand Down Expand Up @@ -629,13 +724,22 @@ export const makeT3Session = (botConfig: DiscordBotConfig) =>

return T3Session.of({
connect,
connectUntilReady,
setOnReconnected: (handler) => {
onReconnected = handler;
},
isConnected: () => Effect.sync(() => session !== null),
isReady: () => Effect.sync(() => session !== null && shell !== null),
shell: () => Effect.succeed(shell),
serverConfig: () => Effect.succeed(serverConfig),
findProjectByWorkspaceRoot: (workspaceRoot) =>
Effect.sync(() => {
Effect.gen(function* () {
// Mentions can land while shell is still filling after reconnect.
for (let attempt = 0; attempt < PROJECT_LOOKUP_SHELL_WAIT_ATTEMPTS; attempt += 1) {
// shell/session are mutated by connect/teardown/shellFiber, not this loop body.
if (shell !== null || session === null) break;
yield* Effect.sleep(Duration.millis(SHELL_SNAPSHOT_POLL_MS));
}
if (shell === null) return null;
const target = normalizeWorkspacePath(workspaceRoot);
return (
Expand Down
9 changes: 9 additions & 0 deletions docs/integrations/discord-bot.md
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,15 @@ finalize only deletes stored stream tip ids — it never scans the channel for o
markers. Design notes:
[`apps/discord-bot/DESIGN-thread-restore.md`](../../apps/discord-bot/DESIGN-thread-restore.md).

**Guest restart / Discord-before-T3 race** (implemented): the Discord gateway can come READY
before the guest T3 process is listening. Boot uses `connectUntilReady` (same backoff as
mid-life reconnect) and does **not** exit the process when T3 is late. Connect is only
considered successful once the orchestration **shell snapshot** has arrived (so project
lookups are not empty). Mid-life drops still tear down + reconnect; transport errors on
dispatch also force reconnect when `RpcSession.closed` is slow/missed. Mentions that land
while T3 is reconnecting get a transient “still connecting… try again” reply instead of a
false “no T3 project registered at this path” error.

## Architecture notes

- Project shortName mapping lives **only on the Discord bot** — not in T3 server config/DB.
Expand Down