discord-bot: configure process alert thresholds

apps/discord-bot/package.json

@@ -17,7 +17,8 @@
     "@t3tools/shared": "workspace:*",
     "dfx": "catalog:",
     "effect": "catalog:",
-    "playwright-core": "1.60.0"
+    "playwright-core": "1.60.0",
+    "yaml": "catalog:"
   },
   "devDependencies": {
     "@effect/vitest": "catalog:",

apps/discord-bot/src/alertProcessRules.test.ts

@@ -0,0 +1,80 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeFSP from "node:fs/promises";
+import * as NodeOS from "node:os";
+import * as NodePath from "node:path";
+import { describe, expect, it } from "vite-plus/test";
+
+import {
+  loadAlertProcessRulesFromFileSync,
+  parseAlertProcessRulesDocument,
+} from "./alertProcessRules.ts";
+
+describe("parseAlertProcessRulesDocument", () => {
+  it("parses structured rules with size and duration shorthands", () => {
+    const rules = parseAlertProcessRulesDocument({
+      rules: [
+        {
+          id: "jaeger-linux",
+          match: "jaeger-linux",
+          rss: "4gb",
+          duration: "5m",
+        },
+      ],
+    });
+
+    expect(rules).toEqual([
+      {
+        id: "jaeger-linux",
+        match: "jaeger-linux",
+        rssMbThreshold: 4096,
+        sustainedForMs: 5 * 60_000,
+      },
+    ]);
+  });
+
+  it("accepts cpu-only rules", () => {
+    const rules = parseAlertProcessRulesDocument([
+      {
+        id: "cpu-hot",
+        match: "worker",
+        cpuPercentThreshold: 90,
+        sustainedFor: "2m",
+      },
+    ]);
+
+    expect(rules[0]).toEqual({
+      id: "cpu-hot",
+      match: "worker",
+      cpuPercentThreshold: 90,
+      sustainedForMs: 2 * 60_000,
+    });
+  });
+});
+
+describe("loadAlertProcessRulesFromFileSync", () => {
+  it("loads yaml files", async () => {
+    const dir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-bot-alert-rules-"));
+    const yamlPath = NodePath.join(dir, "alert-rules.yaml");
+    await NodeFSP.writeFile(
+      yamlPath,
+      [
+        "rules:",
+        "  - id: jaeger-linux",
+        "    match: jaeger-linux",
+        "    rss: 4gb",
+        "    duration: 5m",
+        "",
+      ].join("\n"),
+      "utf8",
+    );
+
+    expect(loadAlertProcessRulesFromFileSync(yamlPath)).toEqual([
+      {
+        id: "jaeger-linux",
+        match: "jaeger-linux",
+        rssMbThreshold: 4096,
+        sustainedForMs: 5 * 60_000,
+      },
+    ]);
+  });
+});

apps/discord-bot/src/alertProcessRules.ts

@@ -0,0 +1,162 @@
+// @effect-diagnostics nodeBuiltinImport:off
+import * as NodeFS from "node:fs";
+import * as NodePath from "node:path";
+import { parse as parseYaml } from "yaml";
+
+import { expandHomePath } from "./projectAliases.ts";
+
+export interface AlertProcessRule {
+  readonly id: string;
+  readonly match: string;
+  readonly rssMbThreshold?: number;
+  readonly cpuPercentThreshold?: number;
+  readonly sustainedForMs: number;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function normalizeNonEmptyString(value: unknown): string | null {
+  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
+}
+
+function parseCpuPercent(value: unknown, field: string): number | undefined {
+  if (value === undefined) return undefined;
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+    throw new Error(`${field} must be a non-negative number.`);
+  }
+  return value;
+}
+
+function parseSizeToMb(value: unknown, field: string): number | undefined {
+  if (value === undefined) return undefined;
+  if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+    return value;
+  }
+  if (typeof value !== "string") {
+    throw new Error(`${field} must be a non-negative number or size string.`);
+  }
+  const trimmed = value.trim().toLowerCase();
+  const match = /^(?<amount>\d+(?:\.\d+)?)\s*(?<unit>b|kb|kib|mb|mib|gb|gib|tb|tib)?$/u.exec(
+    trimmed,
+  );
+  if (!match?.groups) {
+    throw new Error(`${field} must be a size like 4096, 4gb, or 512mb.`);
+  }
+  const amount = Number(match.groups.amount);
+  const unit = match.groups.unit ?? "mb";
+  const multiplier =
+    unit === "b"
+      ? 1 / (1024 * 1024)
+      : unit === "kb" || unit === "kib"
+        ? 1 / 1024
+        : unit === "mb" || unit === "mib"
+          ? 1
+          : unit === "gb" || unit === "gib"
+            ? 1024
+            : unit === "tb" || unit === "tib"
+              ? 1024 * 1024
+              : null;
+  if (multiplier === null) {
+    throw new Error(`${field} has an unsupported unit.`);
+  }
+  return amount * multiplier;
+}
+
+function parseDurationToMs(value: unknown, field: string): number {
+  if (typeof value === "number" && Number.isFinite(value) && value >= 0) {
+    return value;
+  }
+  if (typeof value !== "string") {
+    throw new Error(`${field} must be a non-negative number of milliseconds or duration string.`);
+  }
+  const trimmed = value.trim().toLowerCase();
+  const match = /^(?<amount>\d+(?:\.\d+)?)\s*(?<unit>ms|s|m|h|d)?$/u.exec(trimmed);
+  if (!match?.groups) {
+    throw new Error(`${field} must be a duration like 5m, 30s, or 60000.`);
+  }
+  const amount = Number(match.groups.amount);
+  const unit = match.groups.unit ?? "ms";
+  const multiplier =
+    unit === "ms"
+      ? 1
+      : unit === "s"
+        ? 1_000
+        : unit === "m"
+          ? 60_000
+          : unit === "h"
+            ? 60 * 60_000
+            : unit === "d"
+              ? 24 * 60 * 60_000
+              : null;
+  if (multiplier === null) {
+    throw new Error(`${field} has an unsupported unit.`);
+  }
+  return amount * multiplier;
+}
+
+function parseRule(value: unknown, index: number): AlertProcessRule {
+  if (!isRecord(value)) {
+    throw new Error(`Process alert rule #${index + 1} must be an object.`);
+  }
+  const id = normalizeNonEmptyString(value.id);
+  if (id === null) {
+    throw new Error(`Process alert rule #${index + 1} must include a non-empty id.`);
+  }
+  const match = normalizeNonEmptyString(value.match);
+  if (match === null) {
+    throw new Error(`Process alert rule '${id}' must include a non-empty match string.`);
+  }
+  const rssMbThreshold = parseSizeToMb(
+    value.rssMbThreshold ?? value.rssMb ?? value.rss,
+    `Process alert rule '${id}' rss`,
+  );
+  const cpuPercentThreshold = parseCpuPercent(
+    value.cpuPercentThreshold ?? value.cpuPercent ?? value.cpu,
+    `Process alert rule '${id}' cpu`,
+  );
+  if (rssMbThreshold === undefined && cpuPercentThreshold === undefined) {
+    throw new Error(`Process alert rule '${id}' must set rss and/or cpu thresholds.`);
+  }
+  const sustainedForMs = parseDurationToMs(
+    value.sustainedForMs ?? value.sustainedFor ?? value.duration,
+    `Process alert rule '${id}' duration`,
+  );
+  if (sustainedForMs < 0) {
+    throw new Error(`Process alert rule '${id}' duration must be non-negative.`);
+  }
+  return {
+    id,
+    match,
+    ...(rssMbThreshold === undefined ? {} : { rssMbThreshold }),
+    ...(cpuPercentThreshold === undefined ? {} : { cpuPercentThreshold }),
+    sustainedForMs,
+  };
+}
+
+export function parseAlertProcessRulesDocument(document: unknown): ReadonlyArray<AlertProcessRule> {
+  const source = Array.isArray(document)
+    ? document
+    : isRecord(document) && Array.isArray(document.rules)
+      ? document.rules
+      : null;
+  if (source === null) {
+    throw new Error("Alert process rules file must be an array or an object with a rules array.");
+  }
+  return source.map((entry, index) => parseRule(entry, index));
+}
+
+export function loadAlertProcessRulesFromFileSync(
+  filePath: string | undefined,
+): ReadonlyArray<AlertProcessRule> {
+  if (filePath === undefined || filePath.trim() === "") return [];
+  const resolvedPath = NodePath.resolve(expandHomePath(filePath.trim()));
+  if (!NodeFS.existsSync(resolvedPath)) {
+    throw new Error(`Alert process rules file not found: ${resolvedPath}`);
+  }
+  const raw = NodeFS.readFileSync(resolvedPath, "utf8").trim();
+  if (raw.length === 0) return [];
+  const document = resolvedPath.endsWith(".json") ? JSON.parse(raw) : parseYaml(raw);
+  return parseAlertProcessRulesDocument(document);
+}

apps/discord-bot/src/config.test.ts

@@ -18,6 +18,7 @@ const baseConfig = {
   identityMapPath: undefined,
   honeycombTraceUrlTemplate: undefined,
   alertsChannelId: undefined,
+  alertProcessRulesPath: undefined,
   stateSqlitePath: "/var/lib/t3/userdata/state.sqlite",
   browserEnabled: false,
   browserProfile: "default",

apps/discord-bot/src/config.ts

@@ -39,6 +39,12 @@ export interface DiscordBotConfig {
    * Unset → watchdog does not post.
    */
   readonly alertsChannelId: string | undefined;
+  /**
+   * Optional YAML/JSON rules file for per-process Discord alert ceilings.
+   * Lets operators raise/lower RSS, CPU, and sustained-duration thresholds
+   * for named guest processes without changing code.
+   */
+  readonly alertProcessRulesPath: string | undefined;
   /**
    * Path to t3code `state.sqlite` for long-turn detection (guest: /var/lib/t3/userdata/state.sqlite).
    */
@@ -118,6 +124,10 @@ export const DiscordBotConfig: Effect.Effect<DiscordBotConfig, Config.ConfigErro
       Config.option,
       Config.map(Option.getOrUndefined),
     );
+    const alertProcessRulesPath = yield* Config.string("T3_DISCORD_ALERT_PROCESS_RULES_PATH").pipe(
+      Config.option,
+      Config.map(Option.getOrUndefined),
+    );
     const stateSqlitePath = yield* Config.string("T3_STATE_SQLITE_PATH").pipe(
       Config.withDefault("/var/lib/t3/userdata/state.sqlite"),
     );
@@ -173,6 +183,7 @@ export const DiscordBotConfig: Effect.Effect<DiscordBotConfig, Config.ConfigErro
       identityMapPath,
       honeycombTraceUrlTemplate,
       alertsChannelId,
+      alertProcessRulesPath,
       stateSqlitePath,
       browserEnabled,
       browserProfile,

apps/discord-bot/src/features/Alerts.test.ts

@@ -23,7 +23,7 @@ import {
 const TICK_MS = 60_000;
 const CPU_THRESHOLD = 50;
 const RSS_THRESHOLD = 768;
-const SUSTAINED_TICKS = 5;
+const SUSTAINED_FOR_MS = 4 * TICK_MS;
 
 const proc = (over: Partial<ProcInfo> & { pid: number }): ProcInfo => ({
   pid: over.pid,
@@ -48,9 +48,12 @@ function run(
       prev: state,
       procs,
       nowMs: startMs + index * TICK_MS,
-      cpuPercentThreshold: CPU_THRESHOLD,
-      rssMbThreshold: RSS_THRESHOLD,
-      sustainedTicks: SUSTAINED_TICKS,
+      resolveRule: () => ({
+        id: "default",
+        cpuPercentThreshold: CPU_THRESHOLD,
+        rssMbThreshold: RSS_THRESHOLD,
+        sustainedForMs: SUSTAINED_FOR_MS,
+      }),
     });
     state = result.next;
     hot = result.hot;
@@ -188,23 +191,20 @@ describe("trackSustainedHotProcesses", () => {
 
   it("alerts once a process stays CPU-hot for the sustained window", () => {
     // A full core busy: +60s of CPU per 60s tick = ~100%.
-    const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, (_unused, i) => [
+    const ticks = Array.from({ length: 6 }, (_unused, i) => [
       proc({ pid: 900, rssMb: 200, cpuSeconds: i * 60 }),
     ]);
     const hot = run(ticks).hot;
     expect(hot).toHaveLength(1);
     expect(hot[0]!.pid).toBe(900);
     expect(hot[0]!.cpuPercent).toBeGreaterThanOrEqual(CPU_THRESHOLD);
-    // SUSTAINED_TICKS consecutive hot samples span SUSTAINED_TICKS-1 intervals of
-    // wall time (the first tick only primes the rate baseline).
-    expect(Math.round(hot[0]!.sustainedMs / TICK_MS)).toBe(SUSTAINED_TICKS - 1);
+    expect(Math.round(hot[0]!.sustainedMs / TICK_MS)).toBe(4);
   });
 
   it("does not alert before the sustained window elapses", () => {
-    const ticks = Array.from({ length: SUSTAINED_TICKS }, (_unused, i) => [
+    const ticks = Array.from({ length: 5 }, (_unused, i) => [
       proc({ pid: 900, cpuSeconds: i * 60 }),
     ]);
-    // Only SUSTAINED_TICKS-1 measurable hot ticks so far (first tick has no rate).
     expect(run(ticks).hot).toEqual([]);
   });
 
@@ -217,9 +217,7 @@ describe("trackSustainedHotProcesses", () => {
   });
 
   it("alerts on sustained high RSS even at zero CPU", () => {
-    const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, () => [
-      proc({ pid: 950, rssMb: 900, cpuSeconds: 5 }),
-    ]);
+    const ticks = Array.from({ length: 5 }, () => [proc({ pid: 950, rssMb: 900, cpuSeconds: 5 })]);
     const hot = run(ticks).hot;
     expect(hot).toHaveLength(1);
     expect(hot[0]!.rssMb).toBe(900);