diff --git a/apps/discord-bot/src/features/Alerts.test.ts b/apps/discord-bot/src/features/Alerts.test.ts index 4e5e0dabe71..80d141e1979 100644 --- a/apps/discord-bot/src/features/Alerts.test.ts +++ b/apps/discord-bot/src/features/Alerts.test.ts @@ -5,6 +5,8 @@ import { classifySessionLastError, formatAlertCause, isExpectedSessionLastError, + memoryPolicyForProcess, + parseProcMemoryStatus, selectSessionErrorsForAlert, sessionErrorAlertKey, trackSustainedHotProcesses, @@ -20,6 +22,8 @@ const SUSTAINED_TICKS = 5; const proc = (over: Partial & { pid: number }): ProcInfo => ({ pid: over.pid, rssMb: over.rssMb ?? 100, + rssAnonMb: over.rssAnonMb ?? 50, + rssFileMb: over.rssFileMb ?? 50, cpuSeconds: over.cpuSeconds ?? 0, cmd: over.cmd ?? `/bin/proc-${over.pid}`, label: over.label ?? `proc-${over.pid}`, @@ -42,6 +46,7 @@ function run( nowMs: startMs + index * TICK_MS, cpuPercentThreshold: CPU_THRESHOLD, rssMbThreshold: RSS_THRESHOLD, + memoryPolicyFor: (process) => memoryPolicyForProcess(process, RSS_THRESHOLD), sustainedTicks: SUSTAINED_TICKS, }); state = result.next; @@ -126,6 +131,20 @@ describe("session last_error alert classification", () => { }); describe("trackSustainedHotProcesses", () => { + it("parses total, anonymous, and file-backed RSS from proc status", () => { + expect( + parseProcMemoryStatus(` +VmRSS: 1638400 kB +RssAnon: 921600 kB +RssFile: 716800 kB +`), + ).toEqual({ + rssMb: 1_600, + rssAnonMb: 900, + rssFileMb: 700, + }); + }); + it("does not alert on a long-lived but idle process", () => { // The reported bug: a process with lots of cumulative CPU time that now barely // moves (a few seconds every tick) must never alert. +2s of CPU per 60s tick @@ -173,9 +192,63 @@ describe("trackSustainedHotProcesses", () => { const hot = run(ticks).hot; expect(hot).toHaveLength(1); expect(hot[0]!.rssMb).toBe(900); + expect(hot[0]!.memoryKind).toBe("rss"); expect(hot[0]!.cpuPercent).toBe(0); }); + it("ignores Jaeger file cache below the anonymous-memory threshold", () => { + const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, () => [ + proc({ + pid: 950, + rssMb: 3_000, + rssAnonMb: 900, + rssFileMb: 2_100, + cmd: "/cmd/jaeger/jaeger-linux --config=/etc/jaeger/config.yaml", + }), + ]); + + expect(run(ticks).hot).toEqual([]); + }); + + it("alerts once Jaeger sustains 2 GiB of anonymous memory", () => { + const ticks = Array.from({ length: SUSTAINED_TICKS + 1 }, () => [ + proc({ + pid: 950, + rssMb: 3_000, + rssAnonMb: 2 * 1024, + rssFileMb: 952, + cmd: "/cmd/jaeger/jaeger-linux --config=/etc/jaeger/config.yaml", + }), + ]); + + expect(run(ticks).hot).toEqual([ + expect.objectContaining({ + pid: 950, + memoryKind: "anonymous", + memoryValueMb: 2 * 1024, + memoryThresholdMb: 2 * 1024, + }), + ]); + }); + + it("falls back to total RSS when the kernel omits the Jaeger RSS breakdown", () => { + expect( + memoryPolicyForProcess( + proc({ + pid: 950, + rssMb: 2 * 1024, + rssAnonMb: 0, + rssFileMb: 0, + cmd: "/cmd/jaeger/jaeger-linux", + }), + ), + ).toEqual({ + valueMb: 2 * 1024, + thresholdMb: 2 * 1024, + kind: "rss", + }); + }); + it("treats a reused pid as a new process and resets its streak", () => { const busy = Array.from({ length: 4 }, (_unused, i) => [ proc({ pid: 900, cpuSeconds: i * 60 }), diff --git a/apps/discord-bot/src/features/Alerts.ts b/apps/discord-bot/src/features/Alerts.ts index 2e398a1532c..f8be18303e1 100644 --- a/apps/discord-bot/src/features/Alerts.ts +++ b/apps/discord-bot/src/features/Alerts.ts @@ -38,10 +38,11 @@ const DISK_FREE_MIN_GB = 2; const SENTRY_RSS_ALERT_MB = 512; const SENTRY_COUNT_ALERT = 2; const STUCK_RSS_ALERT_MB = 768; +const JAEGER_ANON_ALERT_MB = 2 * 1024; /** - * A process is "hot" when it holds ≥STUCK_RSS_ALERT_MB of RSS or averages - * ≥SUSTAINED_CPU_PERCENT of a core, and it only alerts once it has stayed hot - * for SUSTAINED_TICKS consecutive ticks. + * A process is "hot" when it exceeds its memory policy or averages at least + * SUSTAINED_CPU_PERCENT of a core, and it only alerts once it has stayed hot for + * SUSTAINED_TICKS consecutive ticks. * * This measures a *rate* (Δcpu / Δwall between ticks), not cumulative CPU time: * a long-lived-but-idle process (e.g. one that gathered 200s of CPU over hours @@ -77,11 +78,38 @@ const RUNAWAY_PATTERNS: ReadonlyArray<{ export interface ProcInfo { readonly pid: number; readonly rssMb: number; + readonly rssAnonMb: number; + readonly rssFileMb: number; readonly cpuSeconds: number; readonly cmd: string; readonly label: string; } +export interface ProcMemoryPolicy { + readonly valueMb: number; + readonly thresholdMb: number; + readonly kind: "anonymous" | "rss"; +} + +export function memoryPolicyForProcess( + proc: ProcInfo, + defaultRssMbThreshold = STUCK_RSS_ALERT_MB, +): ProcMemoryPolicy { + if (/(?:^|[/\s])jaeger(?:$|[/\s-])/i.test(proc.cmd)) { + const hasRssBreakdown = proc.rssAnonMb > 0 || proc.rssFileMb > 0; + return { + valueMb: hasRssBreakdown ? proc.rssAnonMb : proc.rssMb, + thresholdMb: JAEGER_ANON_ALERT_MB, + kind: hasRssBreakdown ? "anonymous" : "rss", + }; + } + return { + valueMb: proc.rssMb, + thresholdMb: defaultRssMbThreshold, + kind: "rss", + }; +} + /** Per-process tracker state carried between ticks to derive a CPU rate. */ export interface ProcSustainState { readonly cpuSeconds: number; @@ -96,6 +124,11 @@ export interface ProcSustainState { export interface SustainedHotProcess { readonly pid: number; readonly rssMb: number; + readonly rssAnonMb: number; + readonly rssFileMb: number; + readonly memoryValueMb: number; + readonly memoryThresholdMb: number; + readonly memoryKind: ProcMemoryPolicy["kind"]; /** Average CPU over the last tick gap, as percent of a single core. */ readonly cpuPercent: number; /** How long it has been continuously hot. */ @@ -116,6 +149,7 @@ export function trackSustainedHotProcesses(input: { readonly nowMs: number; readonly cpuPercentThreshold: number; readonly rssMbThreshold: number; + readonly memoryPolicyFor?: (proc: ProcInfo) => ProcMemoryPolicy; readonly sustainedTicks: number; }): { readonly next: Map; @@ -136,9 +170,14 @@ export function trackSustainedHotProcesses(input: { ? (Math.max(0, proc.cpuSeconds - previous.cpuSeconds) / (elapsedMs / 1_000)) * 100 : null; + const memoryPolicy = input.memoryPolicyFor?.(proc) ?? { + valueMb: proc.rssMb, + thresholdMb: input.rssMbThreshold, + kind: "rss", + }; const isHot = (cpuPercent !== null && cpuPercent >= input.cpuPercentThreshold) || - proc.rssMb >= input.rssMbThreshold; + memoryPolicy.valueMb >= memoryPolicy.thresholdMb; const hotTicks = isHot ? (previous?.hotTicks ?? 0) + 1 : 0; const hotSinceMs = isHot ? previous?.hotTicks @@ -157,6 +196,11 @@ export function trackSustainedHotProcesses(input: { hot.push({ pid: proc.pid, rssMb: proc.rssMb, + rssAnonMb: proc.rssAnonMb, + rssFileMb: proc.rssFileMb, + memoryValueMb: memoryPolicy.valueMb, + memoryThresholdMb: memoryPolicy.thresholdMb, + memoryKind: memoryPolicy.kind, cpuPercent: cpuPercent ?? 0, sustainedMs: input.nowMs - hotSinceMs, label: proc.label, @@ -164,7 +208,7 @@ export function trackSustainedHotProcesses(input: { } } - hot.sort((a, b) => b.cpuPercent - a.cpuPercent || b.rssMb - a.rssMb); + hot.sort((a, b) => b.cpuPercent - a.cpuPercent || b.memoryValueMb - a.memoryValueMb); return { next, hot: hot.slice(0, 8) }; } @@ -354,13 +398,32 @@ function readDisk(path: string): DiskInfo | null { } } -function readRssMb(pid: number): number { +export function parseProcMemoryStatus(status: string): { + readonly rssMb: number; + readonly rssAnonMb: number; + readonly rssFileMb: number; +} { + const readMb = (field: string) => { + const match = new RegExp(`^${field}:\\s+(\\d+)\\s+kB`, "m").exec(status); + return match ? Number(match[1]) / 1024 : 0; + }; + return { + rssMb: readMb("VmRSS"), + rssAnonMb: readMb("RssAnon"), + rssFileMb: readMb("RssFile"), + }; +} + +function readProcMemory(pid: number): { + readonly rssMb: number; + readonly rssAnonMb: number; + readonly rssFileMb: number; +} { try { const status = NodeFS.readFileSync(`/proc/${pid}/status`, "utf8"); - const match = /^VmRSS:\s+(\d+)\s+kB/m.exec(status); - return match ? Number(match[1]) / 1024 : 0; + return parseProcMemoryStatus(status); } catch { - return 0; + return { rssMb: 0, rssAnonMb: 0, rssFileMb: 0 }; } } @@ -414,9 +477,10 @@ function listProcesses(): ReadonlyArray { if (pid === process.pid) continue; const cmd = readCmdline(pid); if (cmd === "") continue; + const memory = readProcMemory(pid); out.push({ pid, - rssMb: readRssMb(pid), + ...memory, cpuSeconds: readCpuSeconds(pid), cmd, label: shortCmd(cmd), @@ -456,6 +520,7 @@ function listFatProcesses( nowMs, cpuPercentThreshold: SUSTAINED_CPU_PERCENT, rssMbThreshold: STUCK_RSS_ALERT_MB, + memoryPolicyFor: (proc) => memoryPolicyForProcess(proc, STUCK_RSS_ALERT_MB), sustainedTicks: SUSTAINED_TICKS, }); sustainState = next; @@ -803,13 +868,20 @@ export const runAlertWatchdog = (botConfig: DiscordBotConfig) => yield* postAlert( "stuck-proc", [ - "**Sustained high RSS / CPU process(es)**", - ...fatNonRunaway.map( - (p) => - `• pid=${p.pid} rss=${p.rssMb.toFixed(0)}MiB cpu≈${p.cpuPercent.toFixed(0)}% ` + - `for ${Math.round(p.sustainedMs / 60_000)}m ${p.label}`, - ), - `_Sustained ≥${SUSTAINED_TICKS} ticks with RSS≥${STUCK_RSS_ALERT_MB}MiB or CPU≥${SUSTAINED_CPU_PERCENT}% of a core (not auto-killed)._`, + "**Sustained high memory / CPU process(es)**", + ...fatNonRunaway.map((p) => { + const memory = + p.memoryKind === "anonymous" + ? `anon=${p.rssAnonMb.toFixed(0)}MiB file-cache=${p.rssFileMb.toFixed(0)}MiB rss=${p.rssMb.toFixed(0)}MiB` + : `rss=${p.rssMb.toFixed(0)}MiB`; + return ( + `• pid=${p.pid} ${memory} ` + + `(memory alert ${p.memoryKind}≥${p.memoryThresholdMb.toFixed(0)}MiB) ` + + `cpu≈${p.cpuPercent.toFixed(0)}% ` + + `for ${Math.round(p.sustainedMs / 60_000)}m ${p.label}` + ); + }), + `_Sustained ≥${SUSTAINED_TICKS} ticks with process-specific high memory or CPU≥${SUSTAINED_CPU_PERCENT}% of a core (not auto-killed)._`, ].join("\n"), ); }