diff --git a/.github/workflows/ops-digest.yml b/.github/workflows/ops-digest.yml new file mode 100644 index 000000000..873779e6e --- /dev/null +++ b/.github/workflows/ops-digest.yml @@ -0,0 +1,102 @@ +# Daily ops digest. Renders the live deep health probe (SLO counters, cache +# hit-rate, answer spend, degraded/truncation rates) into a one-screen summary +# and keeps it in a rolling GitHub issue, commenting only when something is off. +# +# SHIPPED DISABLED: only workflow_dispatch is active. To enable the daily cadence: +# 1. Add repo variable PROD_HEALTH_URL (e.g. https://) +# 2. Add repo secret HEALTH_DEEP_PROBE_SECRET (same value as the deployment) +# 3. Uncomment the `schedule:` block below. +# Run one workflow_dispatch first and confirm the digest issue looks right. +name: Ops Digest + +on: + workflow_dispatch: {} + # schedule: + # # 20:00 UTC = 06:00 Australia/Sydney — a morning digest. + # - cron: "0 20 * * *" + +concurrency: + group: ops-digest + cancel-in-progress: false + +permissions: + contents: read + issues: write + +jobs: + ops-digest: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + PROD_HEALTH_URL: ${{ vars.PROD_HEALTH_URL }} + HEALTH_DEEP_PROBE_SECRET: ${{ secrets.HEALTH_DEEP_PROBE_SECRET }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Preflight config + run: | + missing="" + [ -z "$PROD_HEALTH_URL" ] && missing="$missing PROD_HEALTH_URL(var)" + [ -z "$HEALTH_DEEP_PROBE_SECRET" ] && missing="$missing HEALTH_DEEP_PROBE_SECRET(secret)" + if [ -n "$missing" ]; then + echo "::error::Ops digest cannot run — missing:$missing" + exit 1 + fi + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + + - name: Render digest + id: digest + run: node scripts/ops-digest.mjs --out digest.md + + - name: Publish to rolling issue + uses: actions/github-script@v9 + with: + script: | + const fs = require("fs"); + const label = "ops-digest"; + const body = fs.readFileSync("digest.md", "utf8"); + const status = "${{ steps.digest.outputs.status }}" || "unknown"; + const alerting = "${{ steps.digest.outputs.alerting }}" === "true"; + const { data: existing } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: "open", + labels: label, + }); + let issueNumber; + if (existing.length > 0) { + issueNumber = existing[0].number; + // Keep the issue body as the latest snapshot. + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body, + }); + } else { + const created = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Ops digest", + labels: [label], + body, + }); + issueNumber = created.data.number; + } + // Comment (and therefore notify) only when something needs attention. + if (status !== "ok" || alerting) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber, + body: `⚠ Attention (status=${status}, alerting=${alerting}) — ${new Date().toISOString()}\n\n${body}`, + }); + } + if (status !== "ok") core.warning(`Ops digest status: ${status}`); diff --git a/scripts/ops-digest.mjs b/scripts/ops-digest.mjs new file mode 100644 index 000000000..bb47179ed --- /dev/null +++ b/scripts/ops-digest.mjs @@ -0,0 +1,154 @@ +#!/usr/bin/env node +/** + * ops-digest — render a one-screen operational summary from the live deep health + * probe, for the daily ops-digest workflow. + * + * The app already exposes rich signal at GET /api/health?deep=1 (SLO counters, + * cache hit-rate, answer spend, degraded/truncation rates), but nothing surfaces + * it proactively — a regression is only found when someone looks. This fetches + * that probe and renders Markdown so the workflow can post it to a rolling issue. + * + * Config (all via env, so nothing is hardcoded): + * PROD_HEALTH_URL base URL of the deployment (e.g. https://app.example) + * OR the full health URL; the script appends + * /api/health?deep=1 when only a base is given. + * HEALTH_DEEP_PROBE_SECRET deep-probe token (sent as x-health-deep-token) + * OPS_DIGEST_TIMEOUT_MS fetch timeout (default 20000) + * Flags: + * --out also write the Markdown to a file (for the workflow to read) + * + * Always exits 0 after writing a digest — an unreachable/degraded app is itself + * the report, not a script failure. Emits `status=ok|degraded|unreachable` and + * `alerting=true|false` to $GITHUB_OUTPUT when present so the workflow can flag + * the run without re-parsing. + */ +import { appendFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +function argValue(name) { + const i = process.argv.indexOf(name); + return i >= 0 ? process.argv[i + 1] : undefined; +} + +export function resolveHealthUrl(raw) { + if (!raw) return undefined; + const trimmed = raw.trim().replace(/\/+$/, ""); + if (/\/api\/health/.test(trimmed)) return trimmed; + return `${trimmed}/api/health?deep=1`; +} + +function pct(value) { + return typeof value === "number" ? `${(value * 100).toFixed(1)}%` : "—"; +} + +function usd(value) { + return typeof value === "number" ? `$${value.toFixed(2)}` : "—"; +} + +export function renderDigest(health, meta = {}) { + const lines = []; + const stamp = new Date().toISOString(); + const status = health?.status ?? "unreachable"; + const badge = status === "ok" ? "🟢 ok" : status === "degraded" ? "🟠 degraded" : "🔴 unreachable"; + lines.push(`### Ops digest — ${stamp}`, "", `**Status:** ${badge}`); + if (meta.error) lines.push("", `> Probe error: \`${meta.error}\``); + + if (health) { + if (typeof health.uptimeSeconds === "number") { + const h = Math.floor(health.uptimeSeconds / 3600); + lines.push(`**Uptime:** ${h}h · **Demo mode:** ${health.demoMode ? "yes" : "no"}`); + } + if (health.checks) { + const checks = Object.entries(health.checks) + .map(([k, v]) => `${k}=${v}`) + .join(", "); + lines.push(`**Checks:** ${checks}`); + } + + if (health.slo) { + const s = health.slo; + lines.push( + "", + "**Answer SLO** (trailing " + (s.windowMinutes ?? "?") + "m)", + `- queries: ${s.totalQueries ?? 0}`, + `- hybrid RPC errors: ${s.hybridRpcErrorQueries ?? 0} (${pct(s.hybridRpcErrorRate)})`, + `- degraded/source-only: ${s.degradedQueries ?? 0} (${pct(s.degradedRate)})`, + `- truncation fallbacks: ${s.truncationFallbackQueries ?? 0} (${pct(s.truncationFallbackRate)})`, + `- timeout fallbacks: ${s.timeoutFallbackQueries ?? 0} (${pct(s.timeoutFallbackRate)})`, + ); + } + + if (health.cache) { + const c = health.cache; + lines.push("", `**Cache:** ${c.hits ?? 0}/${c.lookups ?? 0} hits (${pct(c.hitRate)})`); + } + + if (health.spend) { + const sp = health.spend; + const routes = Object.entries(sp.usdByRoute ?? {}) + .map(([r, v]) => `${r} ${usd(v)}`) + .join(", "); + lines.push( + "", + `**Spend** (trailing ${sp.windowMinutes ?? "?"}m): ${usd(sp.usd)} · answers ${sp.answers ?? 0}`, + `- projected/day: ${usd(sp.projectedDailyUsd)}${ + sp.alertDailyUsdThreshold ? ` (threshold ${usd(sp.alertDailyUsdThreshold)})` : "" + }${sp.alerting ? " ⚠ OVER THRESHOLD" : ""}`, + routes ? `- by route: ${routes}` : "", + sp.sampleTruncated ? "- ⚠ sample truncated — figures are a lower bound" : "", + ); + } + } + + lines.push("", "_Read-only deep-probe snapshot. Enable/adjust in `.github/workflows/ops-digest.yml`._"); + return lines.filter((l) => l !== "").join("\n") + "\n"; +} + +async function main() { + const url = resolveHealthUrl(process.env.PROD_HEALTH_URL); + const token = process.env.HEALTH_DEEP_PROBE_SECRET; + const timeoutMs = Number.parseInt(process.env.OPS_DIGEST_TIMEOUT_MS ?? "20000", 10) || 20000; + + let health = null; + let error; + if (!url) { + error = "PROD_HEALTH_URL not set"; + } else { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetch(url, { + headers: token ? { "x-health-deep-token": token } : {}, + signal: controller.signal, + }); + const text = await res.text(); + try { + health = JSON.parse(text); + } catch { + error = `non-JSON response (HTTP ${res.status})`; + } + } catch (e) { + error = e?.name === "AbortError" ? `timeout after ${timeoutMs}ms` : (e?.message ?? String(e)); + } finally { + clearTimeout(timer); + } + } + + const digest = renderDigest(health, { error }); + process.stdout.write(digest); + const out = argValue("--out"); + if (out) writeFileSync(out, digest); + + const status = health?.status ?? "unreachable"; + const alerting = Boolean(health?.spend?.alerting); + if (process.env.GITHUB_OUTPUT) { + appendFileSync(process.env.GITHUB_OUTPUT, `status=${status}\nalerting=${alerting}\n`); + } + // Always exit 0 — the digest content carries the health signal. + process.exit(0); +} + +const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedDirectly) { + main(); +} diff --git a/tests/ops-digest.test.ts b/tests/ops-digest.test.ts new file mode 100644 index 000000000..369326c16 --- /dev/null +++ b/tests/ops-digest.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { renderDigest, resolveHealthUrl } from "../scripts/ops-digest.mjs"; + +describe("resolveHealthUrl", () => { + it("appends the deep health path to a bare base URL", () => { + expect(resolveHealthUrl("https://app.example")).toBe("https://app.example/api/health?deep=1"); + expect(resolveHealthUrl("https://app.example/")).toBe("https://app.example/api/health?deep=1"); + }); + + it("leaves a full health URL untouched", () => { + expect(resolveHealthUrl("https://app.example/api/health?deep=1")).toBe("https://app.example/api/health?deep=1"); + }); + + it("returns undefined for empty input", () => { + expect(resolveHealthUrl("")).toBeUndefined(); + expect(resolveHealthUrl(undefined)).toBeUndefined(); + }); +}); + +describe("renderDigest", () => { + it("renders an unreachable digest when the probe failed", () => { + const md = renderDigest(null, { error: "timeout after 20000ms" }); + expect(md).toContain("unreachable"); + expect(md).toContain("timeout after 20000ms"); + }); + + it("renders SLO, cache, and spend sections from a healthy payload", () => { + const md = renderDigest({ + status: "ok", + demoMode: false, + uptimeSeconds: 7200, + checks: { supabase: "ok", supabaseConfig: "ok" }, + slo: { + windowMinutes: 60, + totalQueries: 100, + hybridRpcErrorQueries: 0, + hybridRpcErrorRate: 0, + degradedQueries: 4, + degradedRate: 0.04, + truncationFallbackQueries: 1, + truncationFallbackRate: 0.01, + timeoutFallbackQueries: 0, + timeoutFallbackRate: 0, + }, + cache: { lookups: 50, hits: 40, hitRate: 0.8 }, + spend: { + windowMinutes: 60, + answers: 100, + usd: 1.23, + usdByRoute: { fast: 0.4, strong: 0.83 }, + projectedDailyUsd: 29.52, + alertDailyUsdThreshold: 50, + alerting: false, + sampleTruncated: false, + }, + }); + expect(md).toContain("🟢 ok"); + expect(md).toContain("Answer SLO"); + expect(md).toContain("degraded/source-only: 4 (4.0%)"); + expect(md).toContain("Cache:** 40/50 hits (80.0%)"); + expect(md).toContain("Spend"); + expect(md).toContain("$1.23"); + expect(md).toContain("projected/day: $29.52"); + expect(md).toContain("fast $0.40"); + }); + + it("flags an over-threshold spend and a truncated sample", () => { + const md = renderDigest({ + status: "ok", + spend: { + windowMinutes: 60, + answers: 5000, + usd: 200, + usdByRoute: {}, + projectedDailyUsd: 4800, + alertDailyUsdThreshold: 100, + alerting: true, + sampleTruncated: true, + }, + }); + expect(md).toContain("OVER THRESHOLD"); + expect(md).toContain("sample truncated"); + }); +});