feat(alerts): Bridge alerting to PagerDuty / Slack / Discord [ENG-361]

docs/bridge-integration/ALERTING.md

@@ -0,0 +1,91 @@
+# Bridge Alerting (ENG-361)
+
+Operational alerting for the Bridge integration. When a Bridge signal fails
+(webhook processing, ERPNext audit write, or a Bridge API outage), the
+`AlertService` (`src/services/alerts`) fans the alert out to the configured
+destinations.
+
+## Routing
+
+| Severity     | PagerDuty (page) | Slack / Mattermost (inform) | Discord (inform) |
+| ------------ | :--------------: | :-------------------------: | :--------------: |
+| **critical** |        ✅        |             ✅              |        ✅        |
+| **warning**  |        —         |             ✅              |        ✅        |
+
+Delivery is best-effort and fire-and-forget — a failing or unconfigured
+destination never blocks or fails the webhook/request path. **A destination
+with no configured credential is silently skipped**, so channels can be enabled
+incrementally.
+
+### Deduplication
+
+Alerts carry a stable `dedupKey` so repeated failures do not spam on-call or chat:
+
+| Destination | Behavior |
+| ----------- | -------- |
+| **PagerDuty** | Events API v2 `dedup_key` groups triggers into one incident |
+| **Slack / Discord** | First message per `dedupKey` within TTL; duplicates are skipped |
+
+Key classes (see `src/services/alerts/dedup-key.ts`):
+
+- `bridge-api:5xx` / `bridge-api:timeout` / `bridge-api:network` — coarse outage keys (30 min inform TTL)
+- `erpnext-audit:deposit:{transfer_id}` — per deposit audit failure (1 h inform TTL)
+- `erpnext-audit:transfer-complete:{transfer_id}` / `transfer-failed:{transfer_id}` — per transfer audit failure
+- `bridge-webhook:deposit:{event_id}` / `bridge-webhook:transfer:{transfer_id}:{event}` — per webhook processing error
+- `ibex:crypto-receive:{tx_hash}` — per IBEX crypto receive webhook failure (1 h inform TTL)
+- `ibex:reconcile:bridge-without-ibex:{tx_hash}` / `ibex:reconcile:ibex-without-bridge:{tx_hash}` — per reconciliation orphan
+- `ibex:reconcile:failed:{tx_hash}` — reconciliation handler threw
+
+Inform dedup is in-process per pod; PagerDuty dedup is global to the service integration.
+
+## Alert sources
+
+| Source                                            | Severity | Where                                                       |
+| ------------------------------------------------- | -------- | ----------------------------------------------------------- |
+| ERPNext audit-write failure (deposit + transfer)  | critical | `services/bridge/webhook-server/routes/{deposit,transfer}.ts` |
+| Bridge webhook processing exception               | critical | same routes (catch block)                                   |
+| Bridge API outage — 5xx / timeout / network       | critical | `services/bridge/client.ts`                                 |
+| IBEX crypto receive webhook failure               | warning  | `services/ibex/webhook-server/routes/crypto-receive.ts`     |
+| Bridge↔IBEX reconciliation orphan / failure         | warning  | `services/bridge/reconciliation.ts`, deposit/crypto catch |
+
+`4xx` responses from Bridge are normal API rejections and are **not** alerted.
+
+## Configuration
+
+Three optional env vars, each gating one destination:
+
+| Env var                       | Destination         | Value                                       |
+| ----------------------------- | ------------------- | ------------------------------------------- |
+| `ALERT_PAGERDUTY_ROUTING_KEY` | PagerDuty           | Events API v2 **integration / routing key** |
+| `ALERT_SLACK_WEBHOOK_URL`     | Slack or Mattermost | Incoming-webhook URL                        |
+| `ALERT_DISCORD_WEBHOOK_URL`   | Discord             | Channel webhook URL                         |
+
+### How to get each value
+
+**PagerDuty** — `ALERT_PAGERDUTY_ROUTING_KEY`
+1. PagerDuty → **Services** → pick (or create) the service that should page for Bridge.
+2. **Integrations** → **Add integration** → **Events API v2**.
+3. Copy the **Integration Key** — that is the routing key.
+
+**Slack** — `ALERT_SLACK_WEBHOOK_URL`
+1. Create/choose a Slack app → **Incoming Webhooks** → **Activate**.
+2. **Add New Webhook to Workspace** → choose the target channel.
+3. Copy the URL (`https://hooks.slack.com/services/...`).
+   _Mattermost works too_ — it accepts the same `{ text }` payload; use its incoming-webhook URL.
+
+**Discord** — `ALERT_DISCORD_WEBHOOK_URL`
+1. Discord → target channel → **Edit Channel** → **Integrations** → **Webhooks**.
+2. **New Webhook** → name it → **Copy Webhook URL**.
+
+### Where to set them
+
+- **Local dev:** add to `.env` (and `.env.ci` for CI).
+- **Staging / production:** set as environment variables / secrets in the deployment — the same place `MATTERMOST_WEBHOOK_URL` is configured. Treat all three as **secrets**.
+
+> If none are set, alerting is a no-op (no errors, no delivery) — useful until the channels are provisioned.
+
+## Verifying in staging (ENG-361 acceptance)
+
+1. Set at least `ALERT_PAGERDUTY_ROUTING_KEY` and `ALERT_SLACK_WEBHOOK_URL` in staging.
+2. Simulate a Bridge webhook failure (e.g. force an ERPNext audit-write error, or replay a malformed transfer webhook).
+3. Confirm on-call is paged via PagerDuty **and** a message posts to Slack within ~1 minute.

src/config/env.ts

@@ -124,6 +124,10 @@ export const env = createEnv({
 
     MATTERMOST_WEBHOOK_URL: z.string().min(1).optional(),
 
+    ALERT_PAGERDUTY_ROUTING_KEY: z.string().min(1).optional(),
+    ALERT_SLACK_WEBHOOK_URL: z.string().url().optional(),
+    ALERT_DISCORD_WEBHOOK_URL: z.string().url().optional(),
+
     PROXY_CHECK_APIKEY: z.string().min(1).optional(),
 
     SVIX_SECRET: z.string().optional(),
@@ -231,6 +235,10 @@ export const env = createEnv({
 
     MATTERMOST_WEBHOOK_URL: process.env.MATTERMOST_WEBHOOK_URL,
 
+    ALERT_PAGERDUTY_ROUTING_KEY: process.env.ALERT_PAGERDUTY_ROUTING_KEY,
+    ALERT_SLACK_WEBHOOK_URL: process.env.ALERT_SLACK_WEBHOOK_URL,
+    ALERT_DISCORD_WEBHOOK_URL: process.env.ALERT_DISCORD_WEBHOOK_URL,
+
     PROXY_CHECK_APIKEY: process.env.PROXY_CHECK_APIKEY,
 
     SVIX_SECRET: process.env.SVIX_SECRET,

src/config/index.ts

@@ -188,6 +188,9 @@ export const NEXTCLOUD_URL = env.NEXTCLOUD_URL
 export const NEXTCLOUD_USER = env.NEXTCLOUD_USER
 export const NEXTCLOUD_PASSWORD = env.NEXTCLOUD_PASSWORD
 export const MATTERMOST_WEBHOOK_URL = env.MATTERMOST_WEBHOOK_URL
+export const ALERT_PAGERDUTY_ROUTING_KEY = env.ALERT_PAGERDUTY_ROUTING_KEY
+export const ALERT_SLACK_WEBHOOK_URL = env.ALERT_SLACK_WEBHOOK_URL
+export const ALERT_DISCORD_WEBHOOK_URL = env.ALERT_DISCORD_WEBHOOK_URL
 export const PROXY_CHECK_APIKEY = env.PROXY_CHECK_APIKEY
 export const NOSTR_PRIVATE_KEY = env.NOSTR_PRIVATE_KEY
 

src/services/alerts/dedup-key.ts

@@ -0,0 +1,34 @@
+export const PAGERDUTY_DEDUP_KEY_MAX = 255
+
+const OUTAGE_TTL_MS = 30 * 60 * 1000
+const DEFAULT_TTL_MS = 60 * 60 * 1000
+
+/** TTL for Slack/Discord first-alert suppression per dedup key class. */
+export const informDedupTtlMs = (dedupKey: string): number =>
+  dedupKey.startsWith("bridge-api") ? OUTAGE_TTL_MS : DEFAULT_TTL_MS
+
+export const generateDedupKey = {
+  bridgeApi5xx: () => "bridge-api:5xx",
+  bridgeApiTimeout: () => "bridge-api:timeout",
+  bridgeApiNetwork: () => "bridge-api:network",
+  erpnextDepositAudit: (transferId: string) => `erpnext-audit:deposit:${transferId}`,
+  erpnextTransferCompletedAudit: (transferId: string) =>
+    `erpnext-audit:transfer-complete:${transferId}`,
+  erpnextTransferFailedAudit: (transferId: string) =>
+    `erpnext-audit:transfer-failed:${transferId}`,
+  bridgeWebhookDeposit: (eventId: string) => `bridge-webhook:deposit:${eventId}`,
+  bridgeWebhookTransfer: (transferId: string, event: string) =>
+    `bridge-webhook:transfer:${transferId}:${event}`,
+  ibexCryptoReceive: (txHash: string) => `ibex:crypto-receive:${txHash.toLowerCase()}`,
+  ibexReconcileBridgeWithoutIbex: (txHash: string) =>
+    `ibex:reconcile:bridge-without-ibex:${txHash.toLowerCase()}`,
+  ibexReconcileBridgeWithoutIbexTransfer: (transferId: string) =>
+    `ibex:reconcile:bridge-without-ibex:transfer:${transferId}`,
+  ibexReconcileIbexWithoutBridge: (txHash: string) =>
+    `ibex:reconcile:ibex-without-bridge:${txHash.toLowerCase()}`,
+  ibexReconcileFailed: (txHash: string) =>
+    `ibex:reconcile:failed:${txHash.toLowerCase()}`,
+}
+
+export const normalizeDedupKey = (key: string): string =>
+  key.length <= PAGERDUTY_DEDUP_KEY_MAX ? key : key.slice(0, PAGERDUTY_DEDUP_KEY_MAX)

src/services/alerts/discord.ts

@@ -0,0 +1,34 @@
+import { ALERT_DISCORD_WEBHOOK_URL } from "@config"
+import { ErrorLevel } from "@domain/shared"
+import { recordExceptionInCurrentSpan } from "@services/tracing"
+import axios from "axios"
+
+import { BridgeAlert } from "./index.types"
+
+// Discord caps message content at 2000 chars; leave headroom.
+const DISCORD_CONTENT_MAX = 1900
+
+// Discord incoming webhook ({ content }).
+export const sendDiscord = async (alert: BridgeAlert): Promise<void> => {
+  if (!ALERT_DISCORD_WEBHOOK_URL) return
+
+  const label = alert.severity === "critical" ? "[CRITICAL]" : "[WARNING]"
+  let content = `${label} **Bridge alert** - ${alert.title}\nsource: \`${alert.source}\` | severity: \`${alert.severity}\``
+  if (alert.detail) content += `\n${alert.detail}`
+  if (alert.context) {
+    content += "\n```json\n" + JSON.stringify(alert.context, null, 2) + "\n```"
+  }
+  if (content.length > DISCORD_CONTENT_MAX) {
+    content = content.slice(0, DISCORD_CONTENT_MAX) + "..."
+  }
+
+  try {
+    await axios.post(
+      ALERT_DISCORD_WEBHOOK_URL,
+      { content },
+      { timeout: 5000, headers: { "Content-Type": "application/json" } },
+    )
+  } catch (error) {
+    recordExceptionInCurrentSpan({ error, level: ErrorLevel.Warn })
+  }
+}

src/services/alerts/ibex-bridge-movement.ts

@@ -0,0 +1,89 @@
+import { generateDedupKey } from "./dedup-key"
+
+import { alertBridge } from "./index"
+
+type IbexMovementAlert = {
+  title: string
+  detail?: string
+  context?: Record<string, unknown>
+}
+
+const alertIbexMovement = (dedupKey: string, alert: IbexMovementAlert): void => {
+  alertBridge({
+    dedupKey,
+    source: "ibex",
+    severity: "warning",
+    ...alert,
+  })
+}
+
+export const alertIbexCryptoReceiveFailure = ({
+  txHash,
+  code,
+  title,
+  detail,
+  context,
+}: {
+  txHash: string
+  code: string
+  title: string
+  detail?: string
+  context?: Record<string, unknown>
+}): void => {
+  alertIbexMovement(generateDedupKey.ibexCryptoReceive(txHash), {
+    title,
+    detail,
+    context: { tx_hash: txHash, code, ...context },
+  })
+}
+
+export const alertIbexReconciliationOrphan = ({
+  orphanType,
+  txHash,
+  transferId,
+  reason,
+  context,
+}: {
+  orphanType: "bridge_without_ibex" | "ibex_without_bridge"
+  txHash?: string
+  transferId?: string
+  reason: string
+  context?: Record<string, unknown>
+}): void => {
+  const dedupKey =
+    orphanType === "ibex_without_bridge" && txHash
+      ? generateDedupKey.ibexReconcileIbexWithoutBridge(txHash)
+      : txHash
+        ? generateDedupKey.ibexReconcileBridgeWithoutIbex(txHash)
+        : generateDedupKey.ibexReconcileBridgeWithoutIbexTransfer(transferId ?? "unknown")
+
+  const title =
+    orphanType === "ibex_without_bridge"
+      ? "IBEX crypto receive without matching Bridge deposit"
+      : "Bridge deposit without matching IBEX crypto receive"
+
+  alertIbexMovement(dedupKey, {
+    title,
+    detail: reason,
+    context: {
+      orphan_type: orphanType,
+      tx_hash: txHash,
+      transfer_id: transferId,
+      ...context,
+    },
+  })
+}
+
+export const alertIbexReconciliationFailed = ({
+  txHash,
+  detail,
+}: {
+  txHash: string
+  detail: string
+}): void => {
+  alertIbexMovement(generateDedupKey.ibexReconcileFailed(txHash), {
+    title: "Bridge↔IBEX reconciliation failed",
+    detail,
+    context: { tx_hash: txHash },
+  })
+}

src/services/alerts/index.ts

@@ -0,0 +1,46 @@
+import { sendPagerDuty } from "./pagerduty"
+import { sendSlack } from "./slack"
+import { sendDiscord } from "./discord"
+import { normalizeDedupKey } from "./dedup-key"
+import { claimInformSlot } from "./inform-dedup"
+import { BridgeAlert } from "./index.types"
+
+export * from "./index.types"
+export { generateDedupKey } from "./dedup-key"
+
+/**
+ * Fire-and-forget fan-out of a Bridge alert to the configured destinations
+ * (ENG-361). Returns immediately; delivery is best-effort: each sender catches
+ * its own errors and no-ops when its credential/URL is unset, so it never throws
+ * or rejects into the caller (no need to await or handle it).
+ *
+ * Routing:
+ *   - critical: page on-call (PagerDuty) + inform (Slack/Mattermost, Discord)
+ *   - warning: inform (Slack/Mattermost, Discord) only
+ *
+ * Dedup:
+ *   - PagerDuty: Events API v2 dedup_key groups triggers into one incident.
+ *   - Slack / Discord: first alert per dedup key within TTL only.
+ */
+export const alertBridge = (alert: BridgeAlert): void => {
+  const dedupKey = normalizeDedupKey(alert.dedupKey)
+  const alertWithKey: BridgeAlert = { ...alert, dedupKey }
+
+  const deliver = async () => {
+    const senders: Promise<void>[] = []
+
+    if (claimInformSlot(dedupKey)) {
+      senders.push(sendSlack(alertWithKey), sendDiscord(alertWithKey))
+    }
+
+    if (alert.severity === "critical") {
+      senders.push(sendPagerDuty(alertWithKey))
+    }
+
+    if (senders.length > 0) {
+      await Promise.allSettled(senders)
+    }
+  }
+
+  deliver().catch(() => undefined)
+}

src/services/alerts/index.types.ts

@@ -0,0 +1,14 @@
+// Ops alerting for Bridge integration signals (ENG-361).
+
+export type AlertSeverity = "critical" | "warning"
+
+export type AlertSource = "bridge-webhook" | "bridge-api" | "ibex" | "erpnext-audit"
+
+export interface BridgeAlert {
+  dedupKey: string
+  source: AlertSource
+  severity: AlertSeverity
+  title: string
+  detail?: string
+  context?: Record<string, unknown>
+}

src/services/alerts/inform-dedup.ts

@@ -0,0 +1,35 @@
+import { informDedupTtlMs } from "./dedup-key"
+
+const seenAt = new Map<string, number>()
+
+/**
+ * Returns true when Slack/Discord should fire for this dedup key (first within TTL).
+ * Subsequent duplicates within the TTL are suppressed.
+ */
+export const claimInformSlot = (dedupKey: string, nowMs = Date.now()): boolean => {
+  const ttlMs = informDedupTtlMs(dedupKey)
+  const lastSentAt = seenAt.get(dedupKey)
+
+  if (lastSentAt !== undefined && nowMs - lastSentAt < ttlMs) {
+    return false
+  }
+
+  seenAt.set(dedupKey, nowMs)
+  pruneExpired(nowMs)
+  return true
+}
+
+const pruneExpired = (nowMs: number): void => {
+  if (seenAt.size < 500) return
+
+  for (const [key, sentAt] of seenAt) {
+    if (nowMs - sentAt >= informDedupTtlMs(key)) {
+      seenAt.delete(key)
+    }
+  }
+}
+
+/** Test helper — clears the in-process inform dedup cache. */
+export const resetInformDedup = (): void => {
+  seenAt.clear()
+}