Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions apps/discord-bot/src/features/Alerts.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,26 @@ describe("Discord alert content", () => {
expect(new TextDecoder().decode(delivery.files[0]?.data)).toBe(trace);
}
});

it("puts t3 thread id on the short fatal/bridge Discord message", () => {
const trace = "stack goes in the attachment";
const threadId = "44c2ab99-729b-4999-83cc-3a4dd04432e0";
const channelId = "1532989230219919520";
const fatal = fatalAlertDelivery("T3 thread subscription exited", trace, {
threadId,
channelId,
});
expect(fatal.content).toContain("**FATAL: T3 thread subscription exited**");
expect(fatal.content).toContain(`thread=\`${threadId}\``);
expect(fatal.content).toContain(`channel=\`${channelId}\``);
expect(fatal.content).not.toContain(trace);
expect(new TextDecoder().decode(fatal.files[0]?.data)).toBe(trace);

const bridge = bridgeAlertDelivery("Working heartbeat failed", trace, { threadId });
expect(bridge.content).toContain("**BRIDGE: Working heartbeat failed**");
expect(bridge.content).toContain(`thread=\`${threadId}\``);
expect(bridge.content).not.toContain("channel=");
});
});

describe("session last_error alert classification", () => {
Expand Down
64 changes: 54 additions & 10 deletions apps/discord-bot/src/features/Alerts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -698,12 +698,46 @@ function alertTraceDelivery(content: string, filename: string, trace: string): A
};
}

export function fatalAlertDelivery(title: string, trace: string): AlertTraceDelivery {
return alertTraceDelivery(`**FATAL: ${title}**`, "fatal-trace.txt", trace);
/** Identity lines shown in the short Discord alert body (not only the attachment). */
export type AlertIdentity = {
readonly threadId?: string;
readonly channelId?: string;
};

function alertIdentityLines(identity?: AlertIdentity): ReadonlyArray<string> {
if (identity === undefined) return [];
const lines: string[] = [];
if (identity.threadId !== undefined && identity.threadId.trim() !== "") {
lines.push(`thread=\`${identity.threadId}\``);
}
if (identity.channelId !== undefined && identity.channelId.trim() !== "") {
lines.push(`channel=\`${identity.channelId}\``);
}
return lines;
}

export function fatalAlertDelivery(
title: string,
trace: string,
identity?: AlertIdentity,
): AlertTraceDelivery {
return alertTraceDelivery(
[`**FATAL: ${title}**`, ...alertIdentityLines(identity)].join("\n"),
"fatal-trace.txt",
trace,
);
}

export function bridgeAlertDelivery(title: string, trace: string): AlertTraceDelivery {
return alertTraceDelivery(`**BRIDGE: ${title}**`, "bridge-trace.txt", trace);
export function bridgeAlertDelivery(
title: string,
trace: string,
identity?: AlertIdentity,
): AlertTraceDelivery {
return alertTraceDelivery(
[`**BRIDGE: ${title}**`, ...alertIdentityLines(identity)].join("\n"),
"bridge-trace.txt",
trace,
);
}

export function sessionErrorAlertDelivery(threadId: string, trace: string): AlertTraceDelivery {
Expand Down Expand Up @@ -744,14 +778,19 @@ export function formatAlertCause(cause: unknown, maxLen?: number): string {
* or channel unset. Does not require DiscordREST in the caller — uses the
* watchdog-held poster.
*/
export const postFatalAlert = (key: string, title: string, detail: string) =>
export const postFatalAlert = (
key: string,
title: string,
detail: string,
identity?: AlertIdentity,
) =>
Effect.gen(function* () {
const p = poster;
if (p === null) {
yield* Effect.logError(`Fatal (no alerts channel): ${title}`, { detail });
yield* Effect.logError(`Fatal (no alerts channel): ${title}`, { detail, ...identity });
return;
}
const delivery = fatalAlertDelivery(title, detail);
const delivery = fatalAlertDelivery(title, detail, identity);
yield* p(`fatal:${key}`, delivery.content, FATAL_COOLDOWN_MS, delivery.files);
});

Expand All @@ -760,14 +799,19 @@ export const postFatalAlert = (key: string, title: string, detail: string) =>
* stream/heartbeat Discord errors, and other bridge soft-failures that leave
* Discord threads desynced while T3 still advances.
*/
export const postBridgeAlert = (key: string, title: string, detail: string) =>
export const postBridgeAlert = (
key: string,
title: string,
detail: string,
identity?: AlertIdentity,
) =>
Effect.gen(function* () {
const p = poster;
if (p === null) {
yield* Effect.logError(`Bridge alert (no alerts channel): ${title}`, { detail });
yield* Effect.logError(`Bridge alert (no alerts channel): ${title}`, { detail, ...identity });
return;
}
const delivery = bridgeAlertDelivery(title, detail);
const delivery = bridgeAlertDelivery(title, detail, identity);
yield* p(`bridge:${key}`, delivery.content, BRIDGE_ALERT_COOLDOWN_MS, delivery.files);
});

Expand Down
1 change: 1 addition & 0 deletions apps/discord-bot/src/features/BridgeHub.ts
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ export const makeBridgeHub = (runBridge: BridgeRunner) =>
`bridge:${input.discordChannelId}`,
"Discord bridge fiber failed",
`channel=\`${input.discordChannelId}\` thread=\`${input.t3ThreadId}\`\n${pretty}`,
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
yield* Deferred.succeed(ready, undefined).pipe(Effect.ignore);
}).pipe(Effect.asVoid),
Expand Down
17 changes: 16 additions & 1 deletion apps/discord-bot/src/features/ResponseBridge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
formatTurnResponseStatsLine,
} from "@t3tools/shared/turnResponseStats";
import { Discord, DiscordConfig, DiscordREST, UI } from "dfx";
import * as Cause from "effect/Cause";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
Expand Down Expand Up @@ -4615,6 +4616,7 @@ export const runBridge = (
`thread=\`${input.t3ThreadId}\``,
formatAlertCause(cause),
].join("\n"),
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
}).pipe(Effect.asVoid),
),
Expand Down Expand Up @@ -4757,6 +4759,7 @@ export const runBridge = (
`phase=\`${phase}\``,
pretty,
].join("\n"),
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
}).pipe(Effect.asVoid);

Expand Down Expand Up @@ -5308,6 +5311,7 @@ export const runBridge = (
`failureCount=${failureCount}`,
pretty,
].join("\n"),
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
}
// Do not advance lastDeliveredSequence — delivery lag keeps HTTP reconcile on.
Expand Down Expand Up @@ -5433,6 +5437,7 @@ export const runBridge = (
`thread=\`${input.t3ThreadId}\``,
pretty,
].join("\n"),
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
}).pipe(Effect.asVoid),
),
Expand Down Expand Up @@ -5565,6 +5570,7 @@ export const runBridge = (
`mode=\`${mode}\``,
pretty,
].join("\n"),
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
}).pipe(Effect.asVoid),
),
Expand Down Expand Up @@ -5621,7 +5627,15 @@ export const runBridge = (
Effect.catchCause((cause) =>
Effect.gen(function* () {
// Follower retries internally; this is only if the outer effect is interrupted
// or fails without recovery.
// or fails without recovery. Bot restart / fiber cancel is interrupt-only —
// expected, high volume; do not page #omegent-alerts.
if (Cause.hasInterruptsOnly(cause)) {
yield* Effect.logInfo("Bridge subscribeThread interrupted (no alert)", {
discordChannelId: input.discordChannelId,
t3ThreadId: input.t3ThreadId,
});
return;
}
const pretty = formatAlertCause(cause);
yield* Effect.logError("Bridge subscribeThread exited", {
discordChannelId: input.discordChannelId,
Expand All @@ -5632,6 +5646,7 @@ export const runBridge = (
`subscribe:${input.t3ThreadId}`,
"T3 thread subscription exited",
`channel=\`${input.discordChannelId}\` thread=\`${input.t3ThreadId}\`\n${pretty}`,
{ threadId: input.t3ThreadId, channelId: input.discordChannelId },
);
}).pipe(Effect.asVoid),
),
Expand Down
1 change: 1 addition & 0 deletions apps/discord-bot/src/features/ThreadRestore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,7 @@ export const rehydrateBridges = (source: "boot" | "reconnect") =>
`rehydrate:${link.discordThreadId}`,
"Bridge rehydrate failed",
`source=\`${source}\` channel=\`${link.discordThreadId}\` thread=\`${link.t3ThreadId}\`\n${pretty}`,
{ threadId: link.t3ThreadId, channelId: link.discordThreadId },
);
}).pipe(Effect.asVoid),
),
Expand Down
Loading