feat(ops): daily ops digest from the deep health probe (dispatch-only)

.github/workflows/ops-digest.yml

@@ -0,0 +1,102 @@
+# Daily ops digest. Renders the live deep health probe (SLO counters, cache
+# hit-rate, answer spend, degraded/truncation rates) into a one-screen summary
+# and keeps it in a rolling GitHub issue, commenting only when something is off.
+#
+# SHIPPED DISABLED: only workflow_dispatch is active. To enable the daily cadence:
+#   1. Add repo variable  PROD_HEALTH_URL          (e.g. https://<app-host>)
+#   2. Add repo secret     HEALTH_DEEP_PROBE_SECRET (same value as the deployment)
+#   3. Uncomment the `schedule:` block below.
+# Run one workflow_dispatch first and confirm the digest issue looks right.
+name: Ops Digest
+
+on:
+  workflow_dispatch: {}
+  # schedule:
+  #   # 20:00 UTC = 06:00 Australia/Sydney — a morning digest.
+  #   - cron: "0 20 * * *"
+
+concurrency:
+  group: ops-digest
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  ops-digest:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    env:
+      PROD_HEALTH_URL: ${{ vars.PROD_HEALTH_URL }}
+      HEALTH_DEEP_PROBE_SECRET: ${{ secrets.HEALTH_DEEP_PROBE_SECRET }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Preflight config
+        run: |
+          missing=""
+          [ -z "$PROD_HEALTH_URL" ] && missing="$missing PROD_HEALTH_URL(var)"
+          [ -z "$HEALTH_DEEP_PROBE_SECRET" ] && missing="$missing HEALTH_DEEP_PROBE_SECRET(secret)"
+          if [ -n "$missing" ]; then
+            echo "::error::Ops digest cannot run — missing:$missing"
+            exit 1
+          fi
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v5
+        with:
+          node-version-file: ".nvmrc"
+
+      - name: Render digest
+        id: digest
+        run: node scripts/ops-digest.mjs --out digest.md
+
+      - name: Publish to rolling issue
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require("fs");
+            const label = "ops-digest";
+            const body = fs.readFileSync("digest.md", "utf8");
+            const status = "${{ steps.digest.outputs.status }}" || "unknown";
+            const alerting = "${{ steps.digest.outputs.alerting }}" === "true";
+            const { data: existing } = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              state: "open",
+              labels: label,
+            });
+            let issueNumber;
+            if (existing.length > 0) {
+              issueNumber = existing[0].number;
+              // Keep the issue body as the latest snapshot.
+              await github.rest.issues.update({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issueNumber,
+                body,
+              });
+            } else {
+              const created = await github.rest.issues.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                title: "Ops digest",
+                labels: [label],
+                body,
+              });
+              issueNumber = created.data.number;
+            }
+            // Comment (and therefore notify) only when something needs attention.
+            if (status !== "ok" || alerting) {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: issueNumber,
+                body: `⚠ Attention (status=${status}, alerting=${alerting}) — ${new Date().toISOString()}\n\n${body}`,
+              });
+            }
+            if (status !== "ok") core.warning(`Ops digest status: ${status}`);

scripts/ops-digest.mjs

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+/**
+ * ops-digest — render a one-screen operational summary from the live deep health
+ * probe, for the daily ops-digest workflow.
+ *
+ * The app already exposes rich signal at GET /api/health?deep=1 (SLO counters,
+ * cache hit-rate, answer spend, degraded/truncation rates), but nothing surfaces
+ * it proactively — a regression is only found when someone looks. This fetches
+ * that probe and renders Markdown so the workflow can post it to a rolling issue.
+ *
+ * Config (all via env, so nothing is hardcoded):
+ *   PROD_HEALTH_URL           base URL of the deployment (e.g. https://app.example)
+ *                             OR the full health URL; the script appends
+ *                             /api/health?deep=1 when only a base is given.
+ *   HEALTH_DEEP_PROBE_SECRET  deep-probe token (sent as x-health-deep-token)
+ *   OPS_DIGEST_TIMEOUT_MS     fetch timeout (default 20000)
+ * Flags:
+ *   --out <path>   also write the Markdown to a file (for the workflow to read)
+ *
+ * Always exits 0 after writing a digest — an unreachable/degraded app is itself
+ * the report, not a script failure. Emits `status=ok|degraded|unreachable` and
+ * `alerting=true|false` to $GITHUB_OUTPUT when present so the workflow can flag
+ * the run without re-parsing.
+ */
+import { appendFileSync, writeFileSync } from "node:fs";
+import { pathToFileURL } from "node:url";
+
+function argValue(name) {
+  const i = process.argv.indexOf(name);
+  return i >= 0 ? process.argv[i + 1] : undefined;
+}
+
+export function resolveHealthUrl(raw) {
+  if (!raw) return undefined;
+  const trimmed = raw.trim().replace(/\/+$/, "");
+  if (/\/api\/health/.test(trimmed)) return trimmed;
+  return `${trimmed}/api/health?deep=1`;
+}
+
+function pct(value) {
+  return typeof value === "number" ? `${(value * 100).toFixed(1)}%` : "—";
+}
+
+function usd(value) {
+  return typeof value === "number" ? `$${value.toFixed(2)}` : "—";
+}
+
+export function renderDigest(health, meta = {}) {
+  const lines = [];
+  const stamp = new Date().toISOString();
+  const status = health?.status ?? "unreachable";
+  const badge = status === "ok" ? "🟢 ok" : status === "degraded" ? "🟠 degraded" : "🔴 unreachable";
+  lines.push(`### Ops digest — ${stamp}`, "", `**Status:** ${badge}`);
+  if (meta.error) lines.push("", `> Probe error: \`${meta.error}\``);
+
+  if (health) {
+    if (typeof health.uptimeSeconds === "number") {
+      const h = Math.floor(health.uptimeSeconds / 3600);
+      lines.push(`**Uptime:** ${h}h  ·  **Demo mode:** ${health.demoMode ? "yes" : "no"}`);
+    }
+    if (health.checks) {
+      const checks = Object.entries(health.checks)
+        .map(([k, v]) => `${k}=${v}`)
+        .join(", ");
+      lines.push(`**Checks:** ${checks}`);
+    }
+
+    if (health.slo) {
+      const s = health.slo;
+      lines.push(
+        "",
+        "**Answer SLO** (trailing " + (s.windowMinutes ?? "?") + "m)",
+        `- queries: ${s.totalQueries ?? 0}`,
+        `- hybrid RPC errors: ${s.hybridRpcErrorQueries ?? 0} (${pct(s.hybridRpcErrorRate)})`,
+        `- degraded/source-only: ${s.degradedQueries ?? 0} (${pct(s.degradedRate)})`,
+        `- truncation fallbacks: ${s.truncationFallbackQueries ?? 0} (${pct(s.truncationFallbackRate)})`,
+        `- timeout fallbacks: ${s.timeoutFallbackQueries ?? 0} (${pct(s.timeoutFallbackRate)})`,
+      );
+    }
+
+    if (health.cache) {
+      const c = health.cache;
+      lines.push("", `**Cache:** ${c.hits ?? 0}/${c.lookups ?? 0} hits (${pct(c.hitRate)})`);
+    }
+
+    if (health.spend) {
+      const sp = health.spend;
+      const routes = Object.entries(sp.usdByRoute ?? {})
+        .map(([r, v]) => `${r} ${usd(v)}`)
+        .join(", ");
+      lines.push(
+        "",
+        `**Spend** (trailing ${sp.windowMinutes ?? "?"}m): ${usd(sp.usd)} · answers ${sp.answers ?? 0}`,
+        `- projected/day: ${usd(sp.projectedDailyUsd)}${
+          sp.alertDailyUsdThreshold ? ` (threshold ${usd(sp.alertDailyUsdThreshold)})` : ""
+        }${sp.alerting ? "  ⚠ OVER THRESHOLD" : ""}`,
+        routes ? `- by route: ${routes}` : "",
+        sp.sampleTruncated ? "- ⚠ sample truncated — figures are a lower bound" : "",
+      );
+    }
+  }
+
+  lines.push("", "_Read-only deep-probe snapshot. Enable/adjust in `.github/workflows/ops-digest.yml`._");
+  return lines.filter((l) => l !== "").join("\n") + "\n";
+}
+
+async function main() {
+  const url = resolveHealthUrl(process.env.PROD_HEALTH_URL);
+  const token = process.env.HEALTH_DEEP_PROBE_SECRET;
+  const timeoutMs = Number.parseInt(process.env.OPS_DIGEST_TIMEOUT_MS ?? "20000", 10) || 20000;
+
+  let health = null;
+  let error;
+  if (!url) {
+    error = "PROD_HEALTH_URL not set";
+  } else {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const res = await fetch(url, {
+        headers: token ? { "x-health-deep-token": token } : {},
+        signal: controller.signal,
+      });
+      const text = await res.text();
+      try {
+        health = JSON.parse(text);
+      } catch {
+        error = `non-JSON response (HTTP ${res.status})`;
+      }
+    } catch (e) {
+      error = e?.name === "AbortError" ? `timeout after ${timeoutMs}ms` : (e?.message ?? String(e));
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  const digest = renderDigest(health, { error });
+  process.stdout.write(digest);
+  const out = argValue("--out");
+  if (out) writeFileSync(out, digest);
+
+  const status = health?.status ?? "unreachable";
+  const alerting = Boolean(health?.spend?.alerting);
+  if (process.env.GITHUB_OUTPUT) {
+    appendFileSync(process.env.GITHUB_OUTPUT, `status=${status}\nalerting=${alerting}\n`);
+  }
+  // Always exit 0 — the digest content carries the health signal.
+  process.exit(0);
+}
+
+const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (invokedDirectly) {
+  main();
+}

tests/ops-digest.test.ts

@@ -0,0 +1,84 @@
+import { describe, expect, it } from "vitest";
+import { renderDigest, resolveHealthUrl } from "../scripts/ops-digest.mjs";
+
+describe("resolveHealthUrl", () => {
+  it("appends the deep health path to a bare base URL", () => {
+    expect(resolveHealthUrl("https://app.example")).toBe("https://app.example/api/health?deep=1");
+    expect(resolveHealthUrl("https://app.example/")).toBe("https://app.example/api/health?deep=1");
+  });
+
+  it("leaves a full health URL untouched", () => {
+    expect(resolveHealthUrl("https://app.example/api/health?deep=1")).toBe("https://app.example/api/health?deep=1");
+  });
+
+  it("returns undefined for empty input", () => {
+    expect(resolveHealthUrl("")).toBeUndefined();
+    expect(resolveHealthUrl(undefined)).toBeUndefined();
+  });
+});
+
+describe("renderDigest", () => {
+  it("renders an unreachable digest when the probe failed", () => {
+    const md = renderDigest(null, { error: "timeout after 20000ms" });
+    expect(md).toContain("unreachable");
+    expect(md).toContain("timeout after 20000ms");
+  });
+
+  it("renders SLO, cache, and spend sections from a healthy payload", () => {
+    const md = renderDigest({
+      status: "ok",
+      demoMode: false,
+      uptimeSeconds: 7200,
+      checks: { supabase: "ok", supabaseConfig: "ok" },
+      slo: {
+        windowMinutes: 60,
+        totalQueries: 100,
+        hybridRpcErrorQueries: 0,
+        hybridRpcErrorRate: 0,
+        degradedQueries: 4,
+        degradedRate: 0.04,
+        truncationFallbackQueries: 1,
+        truncationFallbackRate: 0.01,
+        timeoutFallbackQueries: 0,
+        timeoutFallbackRate: 0,
+      },
+      cache: { lookups: 50, hits: 40, hitRate: 0.8 },
+      spend: {
+        windowMinutes: 60,
+        answers: 100,
+        usd: 1.23,
+        usdByRoute: { fast: 0.4, strong: 0.83 },
+        projectedDailyUsd: 29.52,
+        alertDailyUsdThreshold: 50,
+        alerting: false,
+        sampleTruncated: false,
+      },
+    });
+    expect(md).toContain("🟢 ok");
+    expect(md).toContain("Answer SLO");
+    expect(md).toContain("degraded/source-only: 4 (4.0%)");
+    expect(md).toContain("Cache:** 40/50 hits (80.0%)");
+    expect(md).toContain("Spend");
+    expect(md).toContain("$1.23");
+    expect(md).toContain("projected/day: $29.52");
+    expect(md).toContain("fast $0.40");
+  });
+
+  it("flags an over-threshold spend and a truncated sample", () => {
+    const md = renderDigest({
+      status: "ok",
+      spend: {
+        windowMinutes: 60,
+        answers: 5000,
+        usd: 200,
+        usdByRoute: {},
+        projectedDailyUsd: 4800,
+        alertDailyUsdThreshold: 100,
+        alerting: true,
+        sampleTruncated: true,
+      },
+    });
+    expect(md).toContain("OVER THRESHOLD");
+    expect(md).toContain("sample truncated");
+  });
+});