From b3600da880a3d7612a6e8581d0dccd5940764f81 Mon Sep 17 00:00:00 2001 From: T3 Code PR Stack <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Jul 2026 18:07:46 +0200 Subject: [PATCH] discord-bot: configure process alert thresholds --- apps/discord-bot/package.json | 3 +- .../discord-bot/src/alertProcessRules.test.ts | 80 +++++++++ apps/discord-bot/src/alertProcessRules.ts | 162 ++++++++++++++++++ apps/discord-bot/src/config.test.ts | 1 + apps/discord-bot/src/config.ts | 11 ++ apps/discord-bot/src/features/Alerts.test.ts | 24 ++- apps/discord-bot/src/features/Alerts.ts | 97 ++++++++--- apps/discord-bot/src/main.ts | 1 + .../examples/discord-alert-process-rules.yaml | 18 ++ docs/integrations/discord-bot.md | 25 +++ pnpm-lock.yaml | 3 + 11 files changed, 385 insertions(+), 40 deletions(-) create mode 100644 apps/discord-bot/src/alertProcessRules.test.ts create mode 100644 apps/discord-bot/src/alertProcessRules.ts create mode 100644 docs/examples/discord-alert-process-rules.yaml diff --git a/apps/discord-bot/package.json b/apps/discord-bot/package.json index 8161a3ec6af..7750ef02784 100644 --- a/apps/discord-bot/package.json +++ b/apps/discord-bot/package.json @@ -17,7 +17,8 @@ "@t3tools/shared": "workspace:*", "dfx": "catalog:", "effect": "catalog:", - "playwright-core": "1.60.0" + "playwright-core": "1.60.0", + "yaml": "catalog:" }, "devDependencies": { "@effect/vitest": "catalog:", diff --git a/apps/discord-bot/src/alertProcessRules.test.ts b/apps/discord-bot/src/alertProcessRules.test.ts new file mode 100644 index 00000000000..54d864a7f93 --- /dev/null +++ b/apps/discord-bot/src/alertProcessRules.test.ts @@ -0,0 +1,80 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { describe, expect, it } from "vite-plus/test"; + +import { + loadAlertProcessRulesFromFileSync, + parseAlertProcessRulesDocument, +} from "./alertProcessRules.ts"; + +describe("parseAlertProcessRulesDocument", () => { + it("parses structured rules with size and duration shorthands", () => { + const rules = parseAlertProcessRulesDocument({ + rules: [ + { + id: "jaeger-linux", + match: "jaeger-linux", + rss: "4gb", + duration: "5m", + }, + ], + }); + + expect(rules).toEqual([ + { + id: "jaeger-linux", + match: "jaeger-linux", + rssMbThreshold: 4096, + sustainedForMs: 5 * 60_000, + }, + ]); + }); + + it("accepts cpu-only rules", () => { + const rules = parseAlertProcessRulesDocument([ + { + id: "cpu-hot", + match: "worker", + cpuPercentThreshold: 90, + sustainedFor: "2m", + }, + ]); + + expect(rules[0]).toEqual({ + id: "cpu-hot", + match: "worker", + cpuPercentThreshold: 90, + sustainedForMs: 2 * 60_000, + }); + }); +}); + +describe("loadAlertProcessRulesFromFileSync", () => { + it("loads yaml files", async () => { + const dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-bot-alert-rules-")); + const yamlPath = NodePath.join(dir, "alert-rules.yaml"); + await NodeFSP.writeFile( + yamlPath, + [ + "rules:", + " - id: jaeger-linux", + " match: jaeger-linux", + " rss: 4gb", + " duration: 5m", + "", + ].join("\n"), + "utf8", + ); + + expect(loadAlertProcessRulesFromFileSync(yamlPath)).toEqual([ + { + id: "jaeger-linux", + match: "jaeger-linux", + rssMbThreshold: 4096, + sustainedForMs: 5 * 60_000, + }, + ]); + }); +}); diff --git a/apps/discord-bot/src/alertProcessRules.ts b/apps/discord-bot/src/alertProcessRules.ts new file mode 100644 index 00000000000..63771904ffd --- /dev/null +++ b/apps/discord-bot/src/alertProcessRules.ts @@ -0,0 +1,162 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFS from "node:fs"; +import * as NodePath from "node:path"; +import { parse as parseYaml } from "yaml"; + +import { expandHomePath } from "./projectAliases.ts"; + +export interface AlertProcessRule { + readonly id: string; + readonly match: string; + readonly rssMbThreshold?: number; + readonly cpuPercentThreshold?: number; + readonly sustainedForMs: number; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeNonEmptyString(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; +} + +function parseCpuPercent(value: unknown, field: string): number | undefined { + if (value === undefined) return undefined; + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new Error(`${field} must be a non-negative number.`); + } + return value; +} + +function parseSizeToMb(value: unknown, field: string): number | undefined { + if (value === undefined) return undefined; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + return value; + } + if (typeof value !== "string") { + throw new Error(`${field} must be a non-negative number or size string.`); + } + const trimmed = value.trim().toLowerCase(); + const match = /^(?\d+(?:\.\d+)?)\s*(?b|kb|kib|mb|mib|gb|gib|tb|tib)?$/u.exec( + trimmed, + ); + if (!match?.groups) { + throw new Error(`${field} must be a size like 4096, 4gb, or 512mb.`); + } + const amount = Number(match.groups.amount); + const unit = match.groups.unit ?? "mb"; + const multiplier = + unit === "b" + ? 1 / (1024 * 1024) + : unit === "kb" || unit === "kib" + ? 1 / 1024 + : unit === "mb" || unit === "mib" + ? 1 + : unit === "gb" || unit === "gib" + ? 1024 + : unit === "tb" || unit === "tib" + ? 1024 * 1024 + : null; + if (multiplier === null) { + throw new Error(`${field} has an unsupported unit.`); + } + return amount * multiplier; +} + +function parseDurationToMs(value: unknown, field: string): number { + if (typeof value === "number" && Number.isFinite(value) && value >= 0) { + return value; + } + if (typeof value !== "string") { + throw new Error(`${field} must be a non-negative number of milliseconds or duration string.`); + } + const trimmed = value.trim().toLowerCase(); + const match = /^(?\d+(?:\.\d+)?)\s*(?ms|s|m|h|d)?$/u.exec(trimmed); + if (!match?.groups) { + throw new Error(`${field} must be a duration like 5m, 30s, or 60000.`); + } + const amount = Number(match.groups.amount); + const unit = match.groups.unit ?? "ms"; + const multiplier = + unit === "ms" + ? 1 + : unit === "s" + ? 1_000 + : unit === "m" + ? 60_000 + : unit === "h" + ? 60 * 60_000 + : unit === "d" + ? 24 * 60 * 60_000 + : null; + if (multiplier === null) { + throw new Error(`${field} has an unsupported unit.`); + } + return amount * multiplier; +} + +function parseRule(value: unknown, index: number): AlertProcessRule { + if (!isRecord(value)) { + throw new Error(`Process alert rule #${index + 1} must be an object.`); + } + const id = normalizeNonEmptyString(value.id); + if (id === null) { + throw new Error(`Process alert rule #${index + 1} must include a non-empty id.`); + } + const match = normalizeNonEmptyString(value.match); + if (match === null) { + throw new Error(`Process alert rule '${id}' must include a non-empty match string.`); + } + const rssMbThreshold = parseSizeToMb( + value.rssMbThreshold ?? value.rssMb ?? value.rss, + `Process alert rule '${id}' rss`, + ); + const cpuPercentThreshold = parseCpuPercent( + value.cpuPercentThreshold ?? value.cpuPercent ?? value.cpu, + `Process alert rule '${id}' cpu`, + ); + if (rssMbThreshold === undefined && cpuPercentThreshold === undefined) { + throw new Error(`Process alert rule '${id}' must set rss and/or cpu thresholds.`); + } + const sustainedForMs = parseDurationToMs( + value.sustainedForMs ?? value.sustainedFor ?? value.duration, + `Process alert rule '${id}' duration`, + ); + if (sustainedForMs < 0) { + throw new Error(`Process alert rule '${id}' duration must be non-negative.`); + } + return { + id, + match, + ...(rssMbThreshold === undefined ? {} : { rssMbThreshold }), + ...(cpuPercentThreshold === undefined ? {} : { cpuPercentThreshold }), + sustainedForMs, + }; +} + +export function parseAlertProcessRulesDocument(document: unknown): ReadonlyArray { + const source = Array.isArray(document) + ? document + : isRecord(document) && Array.isArray(document.rules) + ? document.rules + : null; + if (source === null) { + throw new Error("Alert process rules file must be an array or an object with a rules array."); + } + return source.map((entry, index) => parseRule(entry, index)); +} + +export function loadAlertProcessRulesFromFileSync( + filePath: string | undefined, +): ReadonlyArray { + if (filePath === undefined || filePath.trim() === "") return []; + const resolvedPath = NodePath.resolve(expandHomePath(filePath.trim())); + if (!NodeFS.existsSync(resolvedPath)) { + throw new Error(`Alert process rules file not found: ${resolvedPath}`); + } + const raw = NodeFS.readFileSync(resolvedPath, "utf8").trim(); + if (raw.length === 0) return []; + const document = resolvedPath.endsWith(".json") ? JSON.parse(raw) : parseYaml(raw); + return parseAlertProcessRulesDocument(document); +} diff --git a/apps/discord-bot/src/config.test.ts b/apps/discord-bot/src/config.test.ts index 5658b794a09..6aa94bca270 100644 --- a/apps/discord-bot/src/config.test.ts +++ b/apps/discord-bot/src/config.test.ts @@ -18,6 +18,7 @@ const baseConfig = { identityMapPath: undefined, honeycombTraceUrlTemplate: undefined, alertsChannelId: undefined, + alertProcessRulesPath: undefined, stateSqlitePath: "/var/lib/t3/userdata/state.sqlite", browserEnabled: false, browserProfile: "default", diff --git a/apps/discord-bot/src/config.ts b/apps/discord-bot/src/config.ts index c70a749a3c6..2741cda27e4 100644 --- a/apps/discord-bot/src/config.ts +++ b/apps/discord-bot/src/config.ts @@ -39,6 +39,12 @@ export interface DiscordBotConfig { * Unset → watchdog does not post. */ readonly alertsChannelId: string | undefined; + /** + * Optional YAML/JSON rules file for per-process Discord alert ceilings. + * Lets operators raise/lower RSS, CPU, and sustained-duration thresholds + * for named guest processes without changing code. + */ + readonly alertProcessRulesPath: string | undefined; /** * Path to t3code `state.sqlite` for long-turn detection (guest: /var/lib/t3/userdata/state.sqlite). */ @@ -118,6 +124,10 @@ export const DiscordBotConfig: Effect.Effect & { pid: number }): ProcInfo => ({ pid: over.pid, @@ -48,9 +48,12 @@ function run( prev: state, procs, nowMs: startMs + index * TICK_MS, - cpuPercentThreshold: CPU_THRESHOLD, - rssMbThreshold: RSS_THRESHOLD, - sustainedTicks: SUSTAINED_TICKS, + resolveRule: () => ({ + id: "default", + cpuPercentThreshold: CPU_THRESHOLD, + rssMbThreshold: RSS_THRESHOLD, + sustainedForMs: SUSTAINED_FOR_MS, + }), }); state = result.next; hot = result.hot; @@ -188,23 +191,20 @@ describe("trackSustainedHotProcesses", () => { it("alerts once a process stays CPU-hot for the sustained window", () => { // A full core busy: +60s of CPU per 60s tick = ~100%. - const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, (_unused, i) => [ + const ticks = Array.from({ length: 6 }, (_unused, i) => [ proc({ pid: 900, rssMb: 200, cpuSeconds: i * 60 }), ]); const hot = run(ticks).hot; expect(hot).toHaveLength(1); expect(hot[0]!.pid).toBe(900); expect(hot[0]!.cpuPercent).toBeGreaterThanOrEqual(CPU_THRESHOLD); - // SUSTAINED_TICKS consecutive hot samples span SUSTAINED_TICKS-1 intervals of - // wall time (the first tick only primes the rate baseline). - expect(Math.round(hot[0]!.sustainedMs / TICK_MS)).toBe(SUSTAINED_TICKS - 1); + expect(Math.round(hot[0]!.sustainedMs / TICK_MS)).toBe(4); }); it("does not alert before the sustained window elapses", () => { - const ticks = Array.from({ length: SUSTAINED_TICKS }, (_unused, i) => [ + const ticks = Array.from({ length: 5 }, (_unused, i) => [ proc({ pid: 900, cpuSeconds: i * 60 }), ]); - // Only SUSTAINED_TICKS-1 measurable hot ticks so far (first tick has no rate). expect(run(ticks).hot).toEqual([]); }); @@ -217,9 +217,7 @@ describe("trackSustainedHotProcesses", () => { }); it("alerts on sustained high RSS even at zero CPU", () => { - const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, () => [ - proc({ pid: 950, rssMb: 900, cpuSeconds: 5 }), - ]); + const ticks = Array.from({ length: 5 }, () => [proc({ pid: 950, rssMb: 900, cpuSeconds: 5 })]); const hot = run(ticks).hot; expect(hot).toHaveLength(1); expect(hot[0]!.rssMb).toBe(900); diff --git a/apps/discord-bot/src/features/Alerts.ts b/apps/discord-bot/src/features/Alerts.ts index 16b29a8eb21..1b9986c5b69 100644 --- a/apps/discord-bot/src/features/Alerts.ts +++ b/apps/discord-bot/src/features/Alerts.ts @@ -16,6 +16,7 @@ import * as Effect from "effect/Effect"; import * as Redacted from "effect/Redacted"; import * as Schedule from "effect/Schedule"; +import { loadAlertProcessRulesFromFileSync, type AlertProcessRule } from "../alertProcessRules.ts"; import type { DiscordBotConfig } from "../config.ts"; import { createMessageWithAttachments, @@ -24,6 +25,7 @@ import { type DiscordUploadFile, } from "../presentation/discordFiles.ts"; +const POLL_MS = 60 * 1000; const POLL = "60 seconds"; const COOLDOWN_MS = 10 * 60 * 1000; /** Fatal errors use a shorter cooldown so distinct keys still surface quickly. */ @@ -46,11 +48,11 @@ const DISK_FREE_MIN_GB = 2; /** Alert when a legacy stdio Sentry MCP process exceeds this RSS. */ const SENTRY_RSS_ALERT_MB = 512; const SENTRY_COUNT_ALERT = 2; -const STUCK_RSS_ALERT_MB = 768; /** - * A process is "hot" when it holds ≥STUCK_RSS_ALERT_MB of RSS or averages - * ≥SUSTAINED_CPU_PERCENT of a core, and it only alerts once it has stayed hot - * for SUSTAINED_TICKS consecutive ticks. + * Default generic process rule for "unexpectedly hot" processes. + * The sustained duration preserves the prior five-sample window semantics: + * the first sample establishes the CPU-rate baseline, then four 60s intervals + * must stay hot before alerting. * * This measures a *rate* (Δcpu / Δwall between ticks), not cumulative CPU time: * a long-lived-but-idle process (e.g. one that gathered 200s of CPU over hours @@ -58,8 +60,9 @@ const STUCK_RSS_ALERT_MB = 768; * forever. What we want to catch is a process actually pegging CPU or memory for * a sustained stretch. */ -const SUSTAINED_CPU_PERCENT = 50; // percent of a single core, averaged over the tick gap -const SUSTAINED_TICKS = 5; // consecutive hot ticks before alerting (~5 min at POLL=60s) +const DEFAULT_PROCESS_CPU_PERCENT = 50; // percent of a single core, averaged over the tick gap +const DEFAULT_PROCESS_RSS_ALERT_MB = 768; +const DEFAULT_PROCESS_SUSTAINED_FOR_MS = 4 * POLL_MS; const TURN_RUNNING_MIN_MS = 15 * 60 * 1000; /** Paths to check for free space (guest rootfs is tiny; data volume is the real store). */ @@ -95,8 +98,7 @@ export interface ProcInfo { export interface ProcSustainState { readonly cpuSeconds: number; readonly sampledAtMs: number; - /** Consecutive ticks this process has been hot; 0 resets the streak. */ - readonly hotTicks: number; + readonly wasHot: boolean; /** When the current hot streak began, for reporting how long it has lasted. */ readonly hotSinceMs: number; } @@ -107,11 +109,29 @@ export interface SustainedHotProcess { readonly rssMb: number; /** Average CPU over the last tick gap, as percent of a single core. */ readonly cpuPercent: number; + readonly ruleId: string; + readonly rssMbThreshold: number | null; + readonly cpuPercentThreshold: number | null; + readonly sustainedForMs: number; /** How long it has been continuously hot. */ readonly sustainedMs: number; readonly label: string; } +interface ResolvedProcessAlertRule { + readonly id: string; + readonly rssMbThreshold: number | null; + readonly cpuPercentThreshold: number | null; + readonly sustainedForMs: number; +} + +const DEFAULT_PROCESS_ALERT_RULE: ResolvedProcessAlertRule = { + id: "default", + rssMbThreshold: DEFAULT_PROCESS_RSS_ALERT_MB, + cpuPercentThreshold: DEFAULT_PROCESS_CPU_PERCENT, + sustainedForMs: DEFAULT_PROCESS_SUSTAINED_FOR_MS, +}; + /** * Advance the per-process hotness tracker by one tick. * @@ -123,9 +143,7 @@ export function trackSustainedHotProcesses(input: { readonly prev: ReadonlyMap; readonly procs: ReadonlyArray; readonly nowMs: number; - readonly cpuPercentThreshold: number; - readonly rssMbThreshold: number; - readonly sustainedTicks: number; + readonly resolveRule: (proc: ProcInfo) => ResolvedProcessAlertRule; }): { readonly next: Map; readonly hot: ReadonlyArray; @@ -134,6 +152,7 @@ export function trackSustainedHotProcesses(input: { const hot: SustainedHotProcess[] = []; for (const proc of input.procs) { + const rule = input.resolveRule(proc); const prior = input.prev.get(proc.pid); // A counter that went backwards means the pid was reused; ignore the prior. const reused = prior !== undefined && proc.cpuSeconds < prior.cpuSeconds; @@ -146,27 +165,28 @@ export function trackSustainedHotProcesses(input: { : null; const isHot = - (cpuPercent !== null && cpuPercent >= input.cpuPercentThreshold) || - proc.rssMb >= input.rssMbThreshold; - const hotTicks = isHot ? (previous?.hotTicks ?? 0) + 1 : 0; - const hotSinceMs = isHot - ? previous?.hotTicks - ? previous.hotSinceMs - : input.nowMs - : input.nowMs; + (rule.cpuPercentThreshold !== null && + cpuPercent !== null && + cpuPercent >= rule.cpuPercentThreshold) || + (rule.rssMbThreshold !== null && proc.rssMb >= rule.rssMbThreshold); + const hotSinceMs = isHot ? (previous?.wasHot ? previous.hotSinceMs : input.nowMs) : input.nowMs; next.set(proc.pid, { cpuSeconds: proc.cpuSeconds, sampledAtMs: input.nowMs, - hotTicks, + wasHot: isHot, hotSinceMs, }); - if (hotTicks >= input.sustainedTicks) { + if (isHot && input.nowMs - hotSinceMs >= rule.sustainedForMs) { hot.push({ pid: proc.pid, rssMb: proc.rssMb, cpuPercent: cpuPercent ?? 0, + ruleId: rule.id, + rssMbThreshold: rule.rssMbThreshold, + cpuPercentThreshold: rule.cpuPercentThreshold, + sustainedForMs: rule.sustainedForMs, sustainedMs: input.nowMs - hotSinceMs, label: proc.label, }); @@ -446,6 +466,7 @@ let sustainState: ReadonlyMap = new Map(); function listFatProcesses( procs: ReadonlyArray, nowMs: number, + rules: ReadonlyArray, ): ReadonlyArray { // Generic sustained high RSS / CPU alerts (never auto-kill). Our own long-lived // services are excluded — they are expected to run hot and are handled by @@ -459,13 +480,27 @@ function listFatProcesses( cmd.includes("cloud-hypervisor") || cmd.includes("virtiofsd"); + const resolveRule = (proc: ProcInfo): ResolvedProcessAlertRule => { + const normalizedCmd = proc.cmd.toLowerCase(); + const normalizedLabel = proc.label.toLowerCase(); + const custom = rules.find((rule) => { + const match = rule.match.toLowerCase(); + return normalizedCmd.includes(match) || normalizedLabel.includes(match); + }); + if (custom === undefined) return DEFAULT_PROCESS_ALERT_RULE; + return { + id: custom.id, + rssMbThreshold: custom.rssMbThreshold ?? null, + cpuPercentThreshold: custom.cpuPercentThreshold ?? null, + sustainedForMs: custom.sustainedForMs, + }; + }; + const { next, hot } = trackSustainedHotProcesses({ prev: sustainState, procs: procs.filter((p) => !skip(p.cmd)), nowMs, - cpuPercentThreshold: SUSTAINED_CPU_PERCENT, - rssMbThreshold: STUCK_RSS_ALERT_MB, - sustainedTicks: SUSTAINED_TICKS, + resolveRule, }); sustainState = next; return hot; @@ -561,6 +596,7 @@ function listFailedSystemdUnits(): ReadonlyArray { export function collectHostSnapshot(input: { readonly stateSqlitePath: string | undefined; readonly nowMs: number; + readonly alertProcessRules: ReadonlyArray; }): HostSnapshot { const mem = readMemMb(); const load = readLoad(); @@ -576,7 +612,7 @@ export function collectHostSnapshot(input: { memAvailableMb: mem.available, disks, runaways: listRunaways(procs), - fatProcesses: listFatProcesses(procs, input.nowMs), + fatProcesses: listFatProcesses(procs, input.nowMs, input.alertProcessRules), longTurns: listLongRunningTurns(db, TURN_RUNNING_MIN_MS), sessionErrors: listSessionErrors(db), failedUnits: listFailedSystemdUnits(), @@ -701,6 +737,7 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) => const rest = yield* DiscordREST; const discordConfig = yield* DiscordConfig.DiscordConfig; + const alertProcessRules = loadAlertProcessRulesFromFileSync(botConfig.alertProcessRulesPath); const lastSent = new Map(); const postAlert: Poster = (key, content, cooldownMs = COOLDOWN_MS, files = []) => @@ -758,6 +795,7 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) => const snap = collectHostSnapshot({ stateSqlitePath: botConfig.stateSqlitePath, nowMs, + alertProcessRules, }); const loadLimit = snap.nproc * LOAD_RATIO; @@ -874,7 +912,14 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) => `• pid=${p.pid} rss=${p.rssMb.toFixed(0)}MiB cpu≈${p.cpuPercent.toFixed(0)}% ` + `for ${Math.round(p.sustainedMs / 60_000)}m ${p.label}`, ), - `_Sustained ≥${SUSTAINED_TICKS} ticks with RSS≥${STUCK_RSS_ALERT_MB}MiB or CPU≥${SUSTAINED_CPU_PERCENT}% of a core (not auto-killed)._`, + ...fatNonRunaway.map((p) => { + const parts = []; + if (p.rssMbThreshold !== null) parts.push(`RSS≥${p.rssMbThreshold.toFixed(0)}MiB`); + if (p.cpuPercentThreshold !== null) { + parts.push(`CPU≥${p.cpuPercentThreshold.toFixed(0)}% of a core`); + } + return `_rule=${p.ruleId}; sustained ≥${Math.round(p.sustainedForMs / 60_000)}m; ${parts.join(" or ")}._`; + }), ].join("\n"), ); } diff --git a/apps/discord-bot/src/main.ts b/apps/discord-bot/src/main.ts index 28723960635..5b92aa1f661 100644 --- a/apps/discord-bot/src/main.ts +++ b/apps/discord-bot/src/main.ts @@ -83,6 +83,7 @@ const program = Effect.gen(function* () { dataDir: botConfig.dataDir, projectAliasesPath: botConfig.projectAliasesPath ?? "(unset)", identityMapPath: botConfig.identityMapPath ?? "(unset)", + alertProcessRulesPath: botConfig.alertProcessRulesPath ?? "(unset)", }); // Force acquisition of MentionRouter + Discord gateway (must not be pruned). diff --git a/docs/examples/discord-alert-process-rules.yaml b/docs/examples/discord-alert-process-rules.yaml new file mode 100644 index 00000000000..ace1bb2031f --- /dev/null +++ b/docs/examples/discord-alert-process-rules.yaml @@ -0,0 +1,18 @@ +# Optional Discord bot per-process alert thresholds. +# Loaded by the Discord bot watchdog only: +# +# export T3_DISCORD_ALERT_PROCESS_RULES_PATH=~/.t3/discord-bot/alert-process-rules.yaml +# +# Rules match case-insensitive substrings against the full command line and +# the shortened label shown in alerts. + +rules: + - id: jaeger-linux + match: jaeger-linux + rss: 4gb + duration: 5m + +# - id: busy-worker +# match: worker.js +# cpu: 90 +# duration: 3m diff --git a/docs/integrations/discord-bot.md b/docs/integrations/discord-bot.md index f660031fb82..24c8954c758 100644 --- a/docs/integrations/discord-bot.md +++ b/docs/integrations/discord-bot.md @@ -23,6 +23,30 @@ export T3_PROJECT_ALIASES_PATH=~/.t3/discord-bot/project-aliases.yaml The bot resolves channel topics → shortName → workspace path, then finds the matching T3 project in the server shell snapshot by `workspaceRoot`. +## Optional per-process alert rules + +Guest ops alerts can also use a dedicated YAML or JSON file for process-specific +RSS / CPU / sustained-duration ceilings: + +```bash +export T3_DISCORD_ALERT_PROCESS_RULES_PATH=~/.t3/discord-bot/alert-process-rules.yaml +``` + +Example: + +```yaml +rules: + - id: jaeger-linux + match: jaeger-linux + rss: 4gb + duration: 5m +``` + +Rules match case-insensitive substrings against the full process command line +and the shortened label shown in Discord alerts. `rss` is interpreted in MiB +units (`4gb` => `4096` MiB). `cpu` is percent of a single core averaged across +poll windows. + ## Channel binding Set the Discord channel **topic** to include: @@ -40,6 +64,7 @@ cd apps/discord-bot export DISCORD_BOT_TOKEN=... export T3_HTTP_BASE_URL=http://127.0.0.1:3773 export T3_PROJECT_ALIASES_PATH=~/.t3/discord-bot/project-aliases.yaml +export T3_DISCORD_ALERT_PROCESS_RULES_PATH=~/.t3/discord-bot/alert-process-rules.yaml # Pair once and reuse a token, or bootstrap: export T3_BOOTSTRAP_CREDENTIAL=... # from local-bootstrap-credential / pairing diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3bb91c4fec1..ddc638fb9a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -198,6 +198,9 @@ importers: playwright-core: specifier: 1.60.0 version: 1.60.0 + yaml: + specifier: ^2.9.0 + version: 2.9.0 devDependencies: '@effect/vitest': specifier: 4.0.0-beta.102