diff --git a/src/app/accounts/business-account-upgrade-request.ts b/src/app/accounts/business-account-upgrade-request.ts index f1a81b3f0..51ac02e56 100644 --- a/src/app/accounts/business-account-upgrade-request.ts +++ b/src/app/accounts/business-account-upgrade-request.ts @@ -1,5 +1,6 @@ import { AccountsRepository, UsersRepository } from "@services/mongoose" import { IdentityRepository } from "@services/kratos" +import { notifyOpsEvent } from "@services/alerts/ops-events" import ErpNext from "@services/frappe/ErpNext" import { @@ -101,5 +102,17 @@ export const createUpgradeRequest = async ( const requestResult = await ErpNext.postUpgradeRequest(req) if (requestResult instanceof Error) return requestResult + notifyOpsEvent({ + flow: "upgrade", + phase: "requested", + status: "pending", + accountId, + meta: { + requestId: requestResult.name, + from: String(account.level), + to: String(input.level), + }, + }) + return { id: requestResult.name, status: initialStatus } as UpgradeStatusResponse } diff --git a/src/app/accounts/update-account-level.ts b/src/app/accounts/update-account-level.ts index 03e7499ad..ae46a2abb 100644 --- a/src/app/accounts/update-account-level.ts +++ b/src/app/accounts/update-account-level.ts @@ -1,5 +1,6 @@ import { AccountLevel } from "@domain/accounts" import { ValidationError } from "@domain/shared" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { AccountsRepository } from "@services/mongoose" export const updateAccountLevel = async ({ @@ -23,7 +24,19 @@ export const updateAccountLevel = async ({ const account = await accountsRepo.findById(id as AccountId) if (account instanceof Error) return account + const previousLevel = account.level account.level = level if (erpParty !== undefined) account.erpParty = erpParty - return accountsRepo.update(account) + const updated = await accountsRepo.update(account) + if (updated instanceof Error) return updated + + notifyOpsEvent({ + flow: "upgrade", + phase: "approved", + status: "success", + accountId: updated.id, + meta: { from: String(previousLevel), to: String(level) }, + }) + + return updated } diff --git a/src/app/accounts/upgrade-device-account.ts b/src/app/accounts/upgrade-device-account.ts index 671a53af2..93ae7b4c4 100644 --- a/src/app/accounts/upgrade-device-account.ts +++ b/src/app/accounts/upgrade-device-account.ts @@ -1,4 +1,5 @@ import { AccountLevel } from "@domain/accounts" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { AccountsRepository, UsersRepository } from "@services/mongoose" export const upgradeAccountFromDeviceToPhone = async ({ @@ -28,5 +29,15 @@ export const upgradeAccountFromDeviceToPhone = async ({ const accountUpdated = await AccountsRepository().update(accountDevice) if (accountUpdated instanceof Error) return accountUpdated + notifyOpsEvent({ + flow: "verification", + phase: "promoted", + status: "success", + accountId: accountUpdated.id, + userId, + phone, + meta: { from: "trial", to: "verified" }, + }) + return accountUpdated } diff --git a/src/app/authentication/email.ts b/src/app/authentication/email.ts index efa643172..07301ae01 100644 --- a/src/app/authentication/email.ts +++ b/src/app/authentication/email.ts @@ -1,4 +1,5 @@ import { AccountAlreadyHasEmailError } from "@domain/authentication/errors" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { AuthWithEmailPasswordlessService } from "@services/kratos" import { baseLogger } from "@services/logger" import { UsersRepository } from "@services/mongoose" @@ -25,6 +26,14 @@ export const addEmailToIdentity = async ({ const emailRegistrationId = await authServiceEmail.sendEmailWithCode({ email }) if (emailRegistrationId instanceof Error) return emailRegistrationId + notifyOpsEvent({ + flow: "verification", + phase: "otp-sent", + status: "pending", + userId, + email, + }) + const user = await UsersRepository().findById(userId) if (user instanceof Error) return user @@ -40,15 +49,39 @@ export const verifyEmail = async ({ }): Promise => { baseLogger.info({ emailRegistrationId }, "RequestVerifyEmail called") + const notifyFailed = (error: Error) => + notifyOpsEvent({ + flow: "verification", + phase: "otp-failed", + status: "failed", + error: error.constructor.name, + meta: { emailFlowId: emailRegistrationId }, + }) + const authServiceEmail = AuthWithEmailPasswordlessService() const res = await authServiceEmail.validateCode({ code, emailFlowId: emailRegistrationId, }) - if (res instanceof Error) return res + if (res instanceof Error) { + notifyFailed(res) + return res + } const user = await UsersRepository().findById(res.kratosUserId) - if (user instanceof Error) return user + if (user instanceof Error) { + notifyFailed(user) + return user + } + + notifyOpsEvent({ + flow: "verification", + phase: "otp-verified", + status: "success", + userId: res.kratosUserId, + email: res.email, + meta: { emailFlowId: emailRegistrationId }, + }) return user } diff --git a/src/app/authentication/login.ts b/src/app/authentication/login.ts index c87b3705c..cf5d89ab6 100644 --- a/src/app/authentication/login.ts +++ b/src/app/authentication/login.ts @@ -22,6 +22,7 @@ import { PhoneAccountAlreadyExistsNeedToSweepFundsError, } from "@services/kratos" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { WalletsRepository } from "@services/mongoose" import { addAttributesToCurrentSpan, @@ -261,6 +262,22 @@ export const loginDeviceUpgradeWithPhone = async ({ ip: IpAddress account: Account }): Promise => { + const notifyUpgradeFailed = ( + phase: string, + error: Error, + meta?: Record, + ) => + notifyOpsEvent({ + flow: "verification", + phase, + status: "failed", + accountId: account.id, + userId: account.kratosUserId, + phone, + error: error.constructor.name, + meta, + }) + { const limitOk = await checkFailedLoginAttemptPerIpLimits(ip) if (limitOk instanceof Error) return limitOk @@ -271,7 +288,10 @@ export const loginDeviceUpgradeWithPhone = async ({ } const validCode = await isPhoneCodeValid({ phone, code }) - if (validCode instanceof Error) return validCode + if (validCode instanceof Error) { + notifyUpgradeFailed("upgrade-failed", validCode) + return validCode + } await rewardFailedLoginAttemptPerIpLimits(ip) await rewardFailedLoginAttemptPerLoginIdentifierLimits(phone) @@ -286,20 +306,29 @@ export const loginDeviceUpgradeWithPhone = async ({ // check if account is upgradeable const phoneMetadata = await isAllowedToOnboard({ ip, phone }) - if (phoneMetadata instanceof Error) return phoneMetadata + if (phoneMetadata instanceof Error) { + notifyUpgradeFailed("upgrade-failed", phoneMetadata) + return phoneMetadata + } const success = await AuthWithUsernamePasswordDeviceIdService().upgradeToPhoneSchema({ phone, userId: account.kratosUserId, }) - if (success instanceof Error) return success + if (success instanceof Error) { + notifyUpgradeFailed("upgrade-failed", success) + return success + } const res = await upgradeAccountFromDeviceToPhone({ userId: account.kratosUserId, phone, phoneMetadata, }) - if (res instanceof Error) return res + if (res instanceof Error) { + notifyUpgradeFailed("upgrade-failed", res) + return res + } return { success } } @@ -323,7 +352,9 @@ export const loginDeviceUpgradeWithPhone = async ({ "login.upgrade.collisionRejected": true, "login.upgrade.collisionHasDeviceBalance": true, }) - return new PhoneAccountAlreadyExistsNeedToSweepFundsError() + const sweepError = new PhoneAccountAlreadyExistsNeedToSweepFundsError() + notifyUpgradeFailed("upgrade-collision", sweepError, { deviceHasBalance: "true" }) + return sweepError } addAttributesToCurrentSpan({ @@ -331,7 +362,9 @@ export const loginDeviceUpgradeWithPhone = async ({ "login.upgrade.collisionHasDeviceBalance": false, }) - return new PhoneAccountAlreadyExistsCannotUpgradeError() + const collisionError = new PhoneAccountAlreadyExistsCannotUpgradeError() + notifyUpgradeFailed("upgrade-collision", collisionError, { deviceHasBalance: "false" }) + return collisionError } export const loginWithDevice = async ({ diff --git a/src/app/authentication/phone.ts b/src/app/authentication/phone.ts index 02fb007dd..02b3a372c 100644 --- a/src/app/authentication/phone.ts +++ b/src/app/authentication/phone.ts @@ -1,5 +1,7 @@ import { PhoneAlreadyExistsError } from "@domain/authentication/errors" +import { notifyOpsEvent } from "@services/alerts/ops-events" + import { isPhoneCodeValid } from "@services/twilio" import { UsersRepository } from "@services/mongoose" @@ -13,7 +15,35 @@ import { rewardFailedLoginAttemptPerLoginIdentifierLimits, } from "./ratelimits" -export const verifyPhone = async ({ +export const verifyPhone = async (args: { + userId: UserId + phone: PhoneNumber + code: PhoneCode + ip: IpAddress +}): Promise => { + const result = await executeVerifyPhone(args) + notifyOpsEvent( + result instanceof Error + ? { + flow: "verification", + phase: "otp-failed", + status: "failed", + userId: args.userId, + phone: args.phone, + error: result.constructor.name, + } + : { + flow: "verification", + phase: "otp-verified", + status: "success", + userId: args.userId, + phone: args.phone, + }, + ) + return result +} + +const executeVerifyPhone = async ({ userId, phone, code, diff --git a/src/app/authentication/request-code.ts b/src/app/authentication/request-code.ts index f2da66640..9a6d3d1b5 100644 --- a/src/app/authentication/request-code.ts +++ b/src/app/authentication/request-code.ts @@ -9,6 +9,7 @@ import { PhoneAlreadyExistsError } from "@domain/authentication/errors" import { NotImplementedError } from "@domain/errors" import { RateLimitConfig } from "@domain/rate-limit" import { RateLimiterExceededError } from "@domain/rate-limit/errors" +import { notifyOpsEvent } from "@services/alerts/ops-events" import Geetest from "@services/geetest" import { AuthWithEmailPasswordlessService } from "@services/kratos" import { baseLogger } from "@services/logger" @@ -104,7 +105,17 @@ export const requestPhoneCodeForAuthedUser = async ({ return true } - return TwilioClient().initiateVerify({ to: phone, channel }) + const verifyResp = await TwilioClient().initiateVerify({ to: phone, channel }) + if (!(verifyResp instanceof Error)) { + notifyOpsEvent({ + flow: "verification", + phase: "otp-sent", + status: "pending", + userId: user.id, + phone, + }) + } + return verifyResp } export const requestEmailCode = async ({ diff --git a/src/app/offers/CashoutManager.ts b/src/app/offers/CashoutManager.ts index a0ef85771..476f3d833 100644 --- a/src/app/offers/CashoutManager.ts +++ b/src/app/offers/CashoutManager.ts @@ -8,6 +8,7 @@ import { UnexpectedIbexResponse } from "@services/ibex/errors" import { getBankOwnerIbexAccount } from "@services/ledger/caching" import { RepositoryError } from "@domain/errors" +import { notifyOpsEvent, toDisplayAmount } from "@services/alerts/ops-events" import { EmailService } from "@services/email" import ErpNext from "@services/frappe/ErpNext" import { BankAccountQueryError, ExchangeRateQueryError } from "@services/frappe/errors" @@ -130,13 +131,48 @@ const CashoutManager = { if (providedWallet.accountId !== settlementWallet.accountId) return new ValidationError("Offer is not good for provided wallet.") + const displayAmount = toDisplayAmount(offer.details.payment.amount) + notifyOpsEvent({ + flow: "cashout", + phase: "initiated", + status: "pending", + accountId: providedWallet.accountId, + amount: displayAmount, + meta: { offerId: id }, + }) + const validOffer = await ValidOffer.from(offer.details) - if (validOffer instanceof Error) return validOffer + if (validOffer instanceof Error) { + notifyOpsEvent({ + flow: "cashout", + phase: "failed", + status: "failed", + accountId: providedWallet.accountId, + amount: displayAmount, + step: "validation", + error: validOffer.constructor.name, + meta: { offerId: id }, + }) + return validOffer + } const executedOffer = await validOffer.execute() if (executedOffer instanceof Error) return executedOffer else { EmailService.sendCashoutInitiatedEmail(executedOffer) + // On a partial failure (payment made but ERPNext submit failed after + // retry) ValidOffer.execute already emitted the terminal failed event — + // exactly one truthful terminal event per cashout. + if (executedOffer.erpSubmitted) { + notifyOpsEvent({ + flow: "cashout", + phase: "succeeded", + status: "success", + accountId: providedWallet.accountId, + amount: displayAmount, + meta: { offerId: id, cashoutId: executedOffer.cashoutId }, + }) + } return executedOffer } }, diff --git a/src/app/offers/ValidOffer.ts b/src/app/offers/ValidOffer.ts index 478c50663..185a35081 100644 --- a/src/app/offers/ValidOffer.ts +++ b/src/app/offers/ValidOffer.ts @@ -5,6 +5,7 @@ import { RepositoryError } from "@domain/errors" import { AccountsRepository, WalletsRepository } from "@services/mongoose" import Ibex from "@services/ibex/client" +import { notifyOpsEvent } from "@services/alerts/ops-events" import ErpNext, { CashoutId } from "@services/frappe/ErpNext" import { CashoutDraftError, CashoutSubmitError } from "@services/frappe/errors" import { baseLogger } from "@services/logger" @@ -44,9 +45,23 @@ class ValidOffer extends Offer { return new ValidOffer(validation) } + private notifyStepFailed(step: string, error: Error): void { + notifyOpsEvent({ + flow: "cashout", + phase: "failed", + status: "failed", + accountId: this.account.id, + step, + error: error.constructor.name, + }) + } + async execute(): Promise { const cashoutId = await ErpNext.draftCashout(this) - if (cashoutId instanceof CashoutDraftError) return cashoutId + if (cashoutId instanceof CashoutDraftError) { + this.notifyStepFailed("draftCashout", cashoutId) + return cashoutId + } if (!Cashout.SkipPayment) { const resp = await Ibex.payInvoice({ @@ -55,6 +70,7 @@ class ValidOffer extends Offer { }) if (resp instanceof IbexError) { baseLogger.error({ resp }, "Failed to pay invoice for cashout") + this.notifyStepFailed("payInvoice", resp) return resp } } else { @@ -70,10 +86,12 @@ class ValidOffer extends Offer { { cashoutId }, "submitCashout failed after retry — manual intervention required", ) + this.notifyStepFailed("submitCashout", submitted) } } + const erpSubmitted = !(submitted instanceof CashoutSubmitError) - return new InitiatedCashout(this, cashoutId) + return new InitiatedCashout(this, cashoutId, erpSubmitted) } } @@ -83,9 +101,14 @@ export class InitiatedCashout { readonly status = PaymentSendStatus.Pending readonly offer: ValidOffer readonly cashoutId: CashoutId + // False when the ERPNext submit failed after retry (payment made, manual + // intervention pending). Informational for callers/ops — GraphQL output is + // unchanged. + readonly erpSubmitted: boolean - constructor(offer: ValidOffer, cashoutId: CashoutId) { + constructor(offer: ValidOffer, cashoutId: CashoutId, erpSubmitted = true) { this.offer = offer this.cashoutId = cashoutId + this.erpSubmitted = erpSubmitted } } diff --git a/src/app/payments/send-intraledger.ts b/src/app/payments/send-intraledger.ts index ded4791b2..722ff354d 100644 --- a/src/app/payments/send-intraledger.ts +++ b/src/app/payments/send-intraledger.ts @@ -6,12 +6,55 @@ import { PaymentSendStatus } from "@domain/bitcoin/lightning" import { checkedToWalletId } from "@domain/wallets" import { MismatchedCurrencyForWalletError } from "@domain/errors" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { addAttributesToCurrentSpan } from "@services/tracing" import { AccountsRepository, WalletsRepository } from "@services/mongoose" import Ibex from "@services/ibex/client" import { UnexpectedIbexResponse } from "@services/ibex/errors" +// Wallet-id intraledger sends are USD/USDT denominated; the unchecked amount +// is in cents (1 USDT = 1 USD). +const usdCentsDisplay = (cents: number) => ({ + value: (Number(cents) / 100).toFixed(2), + currency: "USD", +}) + +const notifyIntraledgerSendResult = ( + args: IntraLedgerPaymentSendWalletIdArgs, + result: PaymentSendStatus | ApplicationError, + amount?: { value: string; currency: string }, +): void => { + const base = { + flow: "transfer" as const, + amount, + meta: { + senderWalletId: args.senderWalletId, + recipientWalletId: args.recipientWalletId, + }, + } + if (result instanceof Error) { + notifyOpsEvent({ + ...base, + phase: "failed", + status: "failed", + error: result.constructor.name, + meta: { ...base.meta, reason: "error-return" }, + }) + } else if (result === PaymentSendStatus.Failure) { + notifyOpsEvent({ + ...base, + phase: "failed", + status: "failed", + meta: { ...base.meta, reason: "status-failure" }, + }) + } else if (result === PaymentSendStatus.Pending) { + notifyOpsEvent({ ...base, phase: "pending", status: "pending" }) + } else { + notifyOpsEvent({ ...base, phase: "succeeded", status: "success" }) + } +} + const intraledgerPaymentSendWalletId = async ({ recipientWalletId: uncheckedRecipientWalletId, amount: uncheckedAmount, @@ -90,14 +133,20 @@ export const intraledgerPaymentSendWalletIdForBtcWallet = async ( args: IntraLedgerPaymentSendWalletIdArgs, ): Promise => { const validated = await validateIsBtcWallet(args.senderWalletId) - return validated instanceof Error ? validated : intraledgerPaymentSendWalletId(args) + const result = + validated instanceof Error ? validated : await intraledgerPaymentSendWalletId(args) + notifyIntraledgerSendResult(args, result) + return result } export const intraledgerPaymentSendWalletIdForUsdWallet = async ( args: IntraLedgerPaymentSendWalletIdArgs, ): Promise => { const validated = await validateIsUsdWallet(args.senderWalletId, { includeUsdt: true }) - return validated instanceof Error ? validated : intraledgerPaymentSendWalletId(args) + const result = + validated instanceof Error ? validated : await intraledgerPaymentSendWalletId(args) + notifyIntraledgerSendResult(args, result, usdCentsDisplay(args.amount)) + return result } const validateIntraledgerPaymentInputs = async ({ diff --git a/src/app/payments/send-lightning.ts b/src/app/payments/send-lightning.ts index 880205c86..d93dd8dd3 100644 --- a/src/app/payments/send-lightning.ts +++ b/src/app/payments/send-lightning.ts @@ -60,6 +60,7 @@ import { DeviceTokensNotRegisteredNotificationsServiceError } from "@domain/noti import { CallbackService } from "@services/svix" import { getCallbackServiceConfig } from "@config" import { CallbackEventType } from "@domain/callback" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { constructPaymentFlowBuilder, @@ -75,7 +76,55 @@ import { reimburseFee } from "./reimburse-fee" const dealer = DealerPriceService() const paymentFlowRepo = PaymentFlowStateRepository(defaultTimeToExpiryInSeconds) -export const payInvoiceByWalletId = async ({ +// Cents for USD/USDT wallets (1 USDT = 1 USD), sats for BTC wallets. +const usdCentsDisplay = (cents: number) => ({ + value: (Number(cents) / 100).toFixed(2), + currency: "USD", +}) +const satsDisplay = (sats: number) => ({ value: String(sats), currency: "sats" }) + +const notifyLightningSendResult = ( + args: { senderWalletId: WalletId; senderAccount: Account }, + result: PaymentSendStatus | ApplicationError, + amount?: { value: string; currency: string }, +): void => { + const base = { + flow: "transfer" as const, + accountId: args.senderAccount.id, + amount, + meta: { senderWalletId: args.senderWalletId }, + } + if (result instanceof Error) { + notifyOpsEvent({ + ...base, + phase: "failed", + status: "failed", + error: result.constructor.name, + meta: { ...base.meta, reason: "error-return" }, + }) + } else if (result === PaymentSendStatus.Failure) { + notifyOpsEvent({ + ...base, + phase: "failed", + status: "failed", + meta: { ...base.meta, reason: "status-failure" }, + }) + } else if (result === PaymentSendStatus.Pending) { + notifyOpsEvent({ ...base, phase: "pending", status: "pending" }) + } else { + notifyOpsEvent({ ...base, phase: "succeeded", status: "success" }) + } +} + +export const payInvoiceByWalletId = async ( + args: PayInvoiceByWalletIdArgs, +): Promise => { + const result = await executePayInvoiceByWalletId(args) + notifyLightningSendResult(args, result) + return result +} + +const executePayInvoiceByWalletId = async ({ uncheckedPaymentRequest, memo, senderWalletId: uncheckedSenderWalletId, @@ -166,14 +215,20 @@ export const payNoAmountInvoiceByWalletIdForBtcWallet = async ( args: PayNoAmountInvoiceByWalletIdArgs, ): Promise => { const validated = await validateIsBtcWallet(args.senderWalletId) - return validated instanceof Error ? validated : payNoAmountInvoiceByWalletId(args) + const result = + validated instanceof Error ? validated : await payNoAmountInvoiceByWalletId(args) + notifyLightningSendResult(args, result, satsDisplay(args.amount)) + return result } export const payNoAmountInvoiceByWalletIdForUsdWallet = async ( args: PayNoAmountInvoiceByWalletIdArgs, ): Promise => { const validated = await validateIsUsdWallet(args.senderWalletId, { includeUsdt: true }) - return validated instanceof Error ? validated : payNoAmountInvoiceByWalletId(args) + const result = + validated instanceof Error ? validated : await payNoAmountInvoiceByWalletId(args) + notifyLightningSendResult(args, result, usdCentsDisplay(args.amount)) + return result } const validateInvoicePaymentInputs = async ({ diff --git a/src/config/env.ts b/src/config/env.ts index 6fd018b1b..cd56b371c 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -129,6 +129,7 @@ export const env = createEnv({ ALERT_PAGERDUTY_ROUTING_KEY: z.string().min(1).optional(), ALERT_SLACK_WEBHOOK_URL: z.string().url().optional(), ALERT_DISCORD_WEBHOOK_URL: z.string().url().optional(), + OPS_DISCORD_WEBHOOK_URL: z.string().url().optional(), PROXY_CHECK_APIKEY: z.string().min(1).optional(), @@ -241,6 +242,7 @@ export const env = createEnv({ ALERT_PAGERDUTY_ROUTING_KEY: process.env.ALERT_PAGERDUTY_ROUTING_KEY, ALERT_SLACK_WEBHOOK_URL: process.env.ALERT_SLACK_WEBHOOK_URL, ALERT_DISCORD_WEBHOOK_URL: process.env.ALERT_DISCORD_WEBHOOK_URL, + OPS_DISCORD_WEBHOOK_URL: process.env.OPS_DISCORD_WEBHOOK_URL, PROXY_CHECK_APIKEY: process.env.PROXY_CHECK_APIKEY, diff --git a/src/config/index.ts b/src/config/index.ts index dd1c8ef9c..575991953 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -193,6 +193,7 @@ export const MATTERMOST_WEBHOOK_URL = env.MATTERMOST_WEBHOOK_URL export const ALERT_PAGERDUTY_ROUTING_KEY = env.ALERT_PAGERDUTY_ROUTING_KEY export const ALERT_SLACK_WEBHOOK_URL = env.ALERT_SLACK_WEBHOOK_URL export const ALERT_DISCORD_WEBHOOK_URL = env.ALERT_DISCORD_WEBHOOK_URL +export const OPS_DISCORD_WEBHOOK_URL = env.OPS_DISCORD_WEBHOOK_URL export const PROXY_CHECK_APIKEY = env.PROXY_CHECK_APIKEY export const NOSTR_PRIVATE_KEY = env.NOSTR_PRIVATE_KEY diff --git a/src/services/alerts/ops-events.ts b/src/services/alerts/ops-events.ts new file mode 100644 index 000000000..4b0ab18f4 --- /dev/null +++ b/src/services/alerts/ops-events.ts @@ -0,0 +1,224 @@ +import { NETWORK, OPS_DISCORD_WEBHOOK_URL } from "@config" +import { ErrorLevel, JMDAmount, USDAmount, USDTAmount } from "@domain/shared" +import { recordExceptionInCurrentSpan } from "@services/tracing" +import axios, { isAxiosError } from "axios" + +/** + * Fire-and-forget ops event feed: posts color-coded Discord embeds to + * OPS_DISCORD_WEBHOOK_URL when users move through key flows (verification, + * upgrade, cashout, deposit, transfer), so the ops channel reads as a funnel. + * + * Delivery is best-effort: no-op when the webhook URL is unset, never throws, + * never awaited by callers (mirrors alertBridge — returns void, all errors are + * swallowed internally). Events are sent sequentially through a small + * in-process FIFO queue; on HTTP 429 the sender honors Discord's retry_after + * once; when the queue overflows, oldest events are dropped and a single + * summary embed is emitted. + */ + +export type OpsFlow = "verification" | "upgrade" | "cashout" | "deposit" | "transfer" + +export type OpsStatus = "success" | "pending" | "failed" + +export interface OpsEvent { + flow: OpsFlow + phase: string // e.g. "otp-sent", "otp-verified", "promoted", "initiated", "succeeded", "failed" + status: OpsStatus // drives embed color: green/amber/red + accountId?: string + userId?: string + deviceId?: string + phone?: string // masked before sending + email?: string // masked before sending + amount?: { value: number | bigint | string; currency: string } + step?: string // e.g. failing cashout step + error?: string // typed domain error name + meta?: Record // extra ids (offerId, txId...) — long ids get truncated +} + +const EMBED_COLOR: Record = { + success: 0x2ecc71, // green + pending: 0xf39c12, // amber + failed: 0xe74c3c, // red +} + +const FLOW_EMOJI: Record = { + verification: "📲", + upgrade: "⬆️", + cashout: "💸", + deposit: "💰", + transfer: "🔁", +} + +const MAX_QUEUE = 50 +const FETCH_TIMEOUT_MS = 3_000 +const MAX_ID_LENGTH = 12 +const ID_PREFIX_LENGTH = 8 + +// --- Masking / formatting helpers (exported for tests) --- + +// +18765550100 -> +1876…00. Never reveals the middle of the number; short +// inputs collapse to a prefix only so first4+last2 can't reconstruct them. +export const maskPhone = (phone: string): string => { + const plus = phone.startsWith("+") ? "+" : "" + const digits = phone.replace(/\D/g, "") + if (digits.length === 0) return `${plus}…` + if (digits.length <= 6) return `${plus}${digits.slice(0, 2)}…` + return `${plus}${digits.slice(0, 4)}…${digits.slice(-2)}` +} + +// jabari@gmail.com -> j***@gmail.com +export const maskEmail = (email: string): string => { + const at = email.indexOf("@") + if (at <= 0) return "***" + return `${email[0]}***@${email.slice(at + 1)}` +} + +// Long ids (mongo ObjectIds, tx hashes...) -> first 8 chars + "…" so the +// channel can still correlate events without leaking full identifiers. +export const truncateId = (id: string): string => + id.length > MAX_ID_LENGTH ? `${id.slice(0, ID_PREFIX_LENGTH)}…` : id + +// Money classes store minor units (cents / USDT micros); embeds should show +// display units ($95.40, not 9540). +export const toDisplayAmount = ( + amount: USDAmount | USDTAmount | JMDAmount, +): { value: string; currency: string } => ({ + value: amount instanceof USDTAmount ? amount.asNumber(2) : amount.asDollars(), + currency: amount.currencyCode, +}) + +const envLabel = (): string => NETWORK ?? process.env.NODE_ENV ?? "unknown" + +const titleCase = (phrase: string): string => + phrase + .split("-") + .map((word) => (word === "otp" ? "OTP" : word)) + .join(" ") + +type DiscordEmbedField = { name: string; value: string; inline: boolean } +type DiscordEmbed = { + title: string + color: number + fields: DiscordEmbedField[] + timestamp: string +} + +export const buildEmbed = (event: OpsEvent): DiscordEmbed => { + const flowTitle = event.flow[0].toUpperCase() + event.flow.slice(1) + const fields: DiscordEmbedField[] = [] + const field = (name: string, value: string | undefined, inline = true) => { + if (value) fields.push({ name, value, inline }) + } + + field("account", event.accountId && truncateId(event.accountId)) + field("user", event.userId && truncateId(event.userId)) + field("device", event.deviceId && truncateId(event.deviceId)) + field("phone", event.phone && maskPhone(event.phone)) + field("email", event.email && maskEmail(event.email)) + if (event.amount) { + field("amount", `${event.amount.value} ${event.amount.currency}`) + } + field("step", event.step) + field("error", event.error) + for (const [key, value] of Object.entries(event.meta ?? {})) { + if (typeof value === "string" && value) field(key, truncateId(value)) + } + field("env", envLabel()) + + return { + title: `${FLOW_EMOJI[event.flow]} ${flowTitle} — ${titleCase(event.phase)}`, + color: EMBED_COLOR[event.status], + fields, + timestamp: new Date().toISOString(), + } +} + +const droppedSummaryEmbed = (dropped: number): DiscordEmbed => ({ + title: `⚠️ Ops events — ${dropped} event${dropped === 1 ? "" : "s"} dropped`, + color: EMBED_COLOR.failed, + fields: [{ name: "env", value: envLabel(), inline: true }], + timestamp: new Date().toISOString(), +}) + +// --- Delivery queue --- + +const queue: OpsEvent[] = [] +let droppedCount = 0 +let draining: Promise | undefined + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +// Discord rate limit responses carry retry_after in seconds (possibly float), +// in the body and/or the Retry-After header. +const retryAfterMs = (error: unknown): number | undefined => { + if (!isAxiosError(error) || error.response?.status !== 429) return undefined + const body = error.response.data as { retry_after?: number } | undefined + const retryAfter = body?.retry_after ?? Number(error.response.headers?.["retry-after"]) + if (typeof retryAfter !== "number" || !Number.isFinite(retryAfter)) return 1_000 + return Math.ceil(retryAfter * 1000) +} + +const postEmbed = async (embed: DiscordEmbed): Promise => { + const post = () => + axios.post( + OPS_DISCORD_WEBHOOK_URL as string, + { embeds: [embed] }, + { timeout: FETCH_TIMEOUT_MS, headers: { "Content-Type": "application/json" } }, + ) + + try { + await post() + } catch (error) { + const waitMs = retryAfterMs(error) + if (waitMs === undefined) { + recordExceptionInCurrentSpan({ error, level: ErrorLevel.Warn }) + return + } + await sleep(waitMs) + try { + await post() + } catch (retryError) { + recordExceptionInCurrentSpan({ error: retryError, level: ErrorLevel.Warn }) + } + } +} + +const drain = async (): Promise => { + try { + while (queue.length > 0) { + const event = queue.shift() + if (event) await postEmbed(buildEmbed(event)) + + if (queue.length === 0 && droppedCount > 0) { + const dropped = droppedCount + droppedCount = 0 + await postEmbed(droppedSummaryEmbed(dropped)) + } + } + } catch (error) { + recordExceptionInCurrentSpan({ error, level: ErrorLevel.Warn }) + } +} + +/** + * Enqueue an ops event for fire-and-forget delivery. Returns immediately and + * never throws; delivery happens sequentially in the background. + */ +export const notifyOpsEvent = (event: OpsEvent): void => { + if (!OPS_DISCORD_WEBHOOK_URL) return + + queue.push(event) + if (queue.length > MAX_QUEUE) { + queue.shift() + droppedCount += 1 + } + + if (!draining) { + draining = drain().finally(() => { + draining = undefined + }) + } +} + +/** Resolves once the delivery queue is idle. Intended for tests. */ +export const opsEventsSettled = (): Promise => draining ?? Promise.resolve() diff --git a/src/services/bridge/webhook-server/routes/deposit.ts b/src/services/bridge/webhook-server/routes/deposit.ts index f89165380..bdae09f96 100644 --- a/src/services/bridge/webhook-server/routes/deposit.ts +++ b/src/services/bridge/webhook-server/routes/deposit.ts @@ -18,6 +18,7 @@ import { reconcileByTxHash } from "@services/bridge/reconciliation" import { writeBridgeDepositRequest } from "@services/frappe/BridgeTransferRequestWriter" import { alertBridge, generateDedupKey } from "@services/alerts" import { alertIbexReconciliationFailed } from "@services/alerts/ibex-bridge-movement" +import { notifyOpsEvent } from "@services/alerts/ops-events" type DepositEventObject = { id: string @@ -143,6 +144,15 @@ export const depositHandler = async (req: Request, res: Response) => { { error: depositLog, event_id, id: obj.id }, "Failed to persist bridge deposit log", ) + notifyOpsEvent({ + flow: "deposit", + phase: "failed", + status: "failed", + step: "persist-deposit-log", + error: depositLog.constructor.name, + amount: { value: String(obj.amount), currency }, + meta: { customerId, transferId: obj.id }, + }) return res.status(500).json({ error: "Failed to persist deposit log" }) } @@ -181,6 +191,15 @@ export const depositHandler = async (req: Request, res: Response) => { detail: auditResult.message, context: { event_id, transfer_id: obj.id }, }) + notifyOpsEvent({ + flow: "deposit", + phase: "failed", + status: "failed", + step: "erpnext-audit", + error: auditResult.constructor.name, + amount: { value: String(obj.amount), currency }, + meta: { customerId, transferId: obj.id }, + }) return res.status(500).json({ error: "Failed to persist ERPNext audit row" }) } @@ -195,6 +214,14 @@ export const depositHandler = async (req: Request, res: Response) => { return res.status(200).json({ status: "already_processed" }) } + notifyOpsEvent({ + flow: "deposit", + phase: "succeeded", + status: "success", + amount: { value: String(obj.amount), currency }, + meta: { customerId, transferId: obj.id }, + }) + return res.status(200).json({ status: "success" }) } catch (error) { baseLogger.error( @@ -209,6 +236,14 @@ export const depositHandler = async (req: Request, res: Response) => { detail: error instanceof Error ? error.message : String(error), context: { event_id, transfer_id: obj.id }, }) + notifyOpsEvent({ + flow: "deposit", + phase: "failed", + status: "failed", + step: "exception", + error: error instanceof Error ? error.constructor.name : String(error), + meta: { transferId: obj.id }, + }) return res.status(500).json({ error: "Internal server error" }) } } diff --git a/src/services/bridge/webhook-server/routes/transfer.ts b/src/services/bridge/webhook-server/routes/transfer.ts index d5ea2402b..44b5c1c92 100644 --- a/src/services/bridge/webhook-server/routes/transfer.ts +++ b/src/services/bridge/webhook-server/routes/transfer.ts @@ -14,6 +14,7 @@ import { writeBridgeCashoutFailed, } from "@services/frappe/BridgeTransferRequestWriter" import { alertBridge, generateDedupKey } from "@services/alerts" +import { notifyOpsEvent } from "@services/alerts/ops-events" const TERMINAL_FAILURE_STATES = new Set([ "undeliverable", @@ -158,6 +159,18 @@ export const transferHandler = async (req: Request, res: Response) => { return res.status(200).json({ status: "already_processed" }) } + notifyOpsEvent({ + flow: "transfer", + phase: "succeeded", + status: "success", + accountId: result.accountId, + amount: { + value: String(result.amount ?? amount ?? ""), + currency: String(result.currency ?? currency ?? ""), + }, + meta: { transferId: transfer_id }, + }) + await sendBridgeWithdrawalNotificationBestEffort({ accountId: result.accountId, amount: result.amount, @@ -241,6 +254,19 @@ export const transferHandler = async (req: Request, res: Response) => { return res.status(200).json({ status: "already_processed" }) } + notifyOpsEvent({ + flow: "transfer", + phase: "failed", + status: "failed", + accountId: result.accountId, + error: result.failureReason ?? failureReason ?? state, + amount: { + value: String(result.amount ?? amount ?? ""), + currency: String(result.currency ?? currency ?? ""), + }, + meta: { transferId: transfer_id }, + }) + await sendBridgeWithdrawalNotificationBestEffort({ accountId: result.accountId, amount: result.amount, diff --git a/src/services/ibex/webhook-server/routes/on-receive.ts b/src/services/ibex/webhook-server/routes/on-receive.ts index 29d2c277a..9fd6c7d43 100644 --- a/src/services/ibex/webhook-server/routes/on-receive.ts +++ b/src/services/ibex/webhook-server/routes/on-receive.ts @@ -1,5 +1,6 @@ import express, { Request, Response, NextFunction } from "express" import rateLimitMiddleware from "express-rate-limit" +import { notifyOpsEvent } from "@services/alerts/ops-events" import { baseLogger, baseLogger as logger } from "@services/logger" import { NotificationsService } from "@services/notifications" @@ -99,6 +100,15 @@ const sendLightningNotification = async ( logger.error(nsResp) } + notifyOpsEvent({ + flow: "deposit", + phase: "succeeded", + status: "success", + accountId: recipientAccount.id, + amount: { value: transaction.amount, currency: receiverWallet.currency }, + meta: { type: "lightning" }, + }) + next() } @@ -138,6 +148,15 @@ const sendOnchainNotification = async ( logger.error(nResp) } + notifyOpsEvent({ + flow: "deposit", + phase: "succeeded", + status: "success", + accountId: recipientAccount.id, + amount: { value: usdAmount.asDollars(), currency: "USD" }, + meta: { type: "onchain", txHash: transaction.hash }, + }) + next() } diff --git a/test/flash/unit/app/accounts/ops-events-hooks.spec.ts b/test/flash/unit/app/accounts/ops-events-hooks.spec.ts new file mode 100644 index 000000000..1b1fbe9ab --- /dev/null +++ b/test/flash/unit/app/accounts/ops-events-hooks.spec.ts @@ -0,0 +1,218 @@ +const mockFindUserById = jest.fn() +const mockUpdateUser = jest.fn() +const mockFindAccountById = jest.fn() +const mockFindAccountByUserId = jest.fn() +const mockUpdateAccount = jest.fn() +const mockGetIdentity = jest.fn() +const mockGetUpgradeRequestList = jest.fn() +const mockCloseUpgradeRequests = jest.fn() +const mockPostUpgradeRequest = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn().mockResolvedValue(undefined), +})) + +jest.mock("@services/mongoose", () => ({ + UsersRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindUserById(...args), + update: (...args: unknown[]) => mockUpdateUser(...args), + })), + AccountsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindAccountById(...args), + findByUserId: (...args: unknown[]) => mockFindAccountByUserId(...args), + update: (...args: unknown[]) => mockUpdateAccount(...args), + })), +})) + +jest.mock("@services/kratos", () => ({ + IdentityRepository: jest.fn(() => ({ + getIdentity: (...args: unknown[]) => mockGetIdentity(...args), + })), +})) + +jest.mock("@services/frappe/ErpNext", () => ({ + __esModule: true, + default: { + getAccountUpgradeRequestList: (...args: unknown[]) => + mockGetUpgradeRequestList(...args), + closeAccountUpgradeRequests: (...args: unknown[]) => + mockCloseUpgradeRequests(...args), + postUpgradeRequest: (...args: unknown[]) => mockPostUpgradeRequest(...args), + }, +})) + +jest.mock("@services/frappe/models/AccountUpgradeRequest", () => { + const actual = jest.requireActual("@services/frappe/models/AccountUpgradeRequest") + return { + ...actual, + AccountUpgradeRequest: jest.fn().mockImplementation(() => ({ + validate: jest.fn(async () => ({ name: "" })), + })), + } +}) + +import { AccountLevel } from "@domain/accounts" +import { upgradeAccountFromDeviceToPhone } from "@app/accounts/upgrade-device-account" +import { updateAccountLevel } from "@app/accounts/update-account-level" +import { createUpgradeRequest } from "@app/accounts/business-account-upgrade-request" +import { notifyOpsEvent } from "@services/alerts/ops-events" + +class RepoBoomError extends Error {} + +const userId = "11111111-1111-4111-8111-111111111111" as UserId +const accountId = "64df1a2b3c4d5e6f78901234" as AccountId +const phone = "+18765550100" as PhoneNumber + +describe("ops events — accounts hooks", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("upgradeAccountFromDeviceToPhone", () => { + it("notifies promoted on success", async () => { + mockFindUserById.mockResolvedValue({ id: userId }) + mockUpdateUser.mockResolvedValue({ id: userId, phone }) + mockFindAccountByUserId.mockResolvedValue({ + id: accountId, + level: AccountLevel.Zero, + }) + mockUpdateAccount.mockImplementation(async (account) => account) + + const result = await upgradeAccountFromDeviceToPhone({ userId, phone }) + + expect(result).not.toBeInstanceOf(Error) + expect((result as Account).level).toBe(AccountLevel.One) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "promoted", + status: "success", + accountId, + userId, + phone, + meta: { from: "trial", to: "verified" }, + }), + ) + }) + + it("does not notify when the account update fails", async () => { + mockFindUserById.mockResolvedValue({ id: userId }) + mockUpdateUser.mockResolvedValue({ id: userId, phone }) + mockFindAccountByUserId.mockResolvedValue({ + id: accountId, + level: AccountLevel.Zero, + }) + mockUpdateAccount.mockResolvedValue(new RepoBoomError("db down")) + + const result = await upgradeAccountFromDeviceToPhone({ userId, phone }) + + expect(result).toBeInstanceOf(RepoBoomError) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) + + describe("updateAccountLevel", () => { + it("notifies approved with old and new level on success", async () => { + mockFindAccountById.mockResolvedValue({ id: accountId, level: AccountLevel.One }) + mockUpdateAccount.mockImplementation(async (account) => account) + + const result = await updateAccountLevel({ + id: accountId, + level: AccountLevel.Two, + erpParty: "CUST-0001", + }) + + expect(result).not.toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "upgrade", + phase: "approved", + status: "success", + accountId, + meta: { from: String(AccountLevel.One), to: String(AccountLevel.Two) }, + }), + ) + }) + + it("does not notify when the update fails", async () => { + mockFindAccountById.mockResolvedValue({ id: accountId, level: AccountLevel.One }) + mockUpdateAccount.mockResolvedValue(new RepoBoomError("db down")) + + const result = await updateAccountLevel({ + id: accountId, + level: AccountLevel.Two, + erpParty: "CUST-0001", + }) + + expect(result).toBeInstanceOf(RepoBoomError) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + + it("does not notify on validation errors", async () => { + const result = await updateAccountLevel({ id: accountId, level: AccountLevel.Two }) + + expect(result).toBeInstanceOf(Error) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) + + describe("createUpgradeRequest", () => { + const input = { + level: AccountLevel.Pro, + accountId, + fullName: "Test Business", + address: { + title: "HQ", + line1: "1 Main St", + city: "Kingston", + state: "St. Andrew", + country: "Jamaica", + }, + terminalsRequested: 1, + idDocument: "doc-1", + } + + beforeEach(() => { + mockFindAccountById.mockResolvedValue({ + id: accountId, + kratosUserId: userId, + username: "testbiz", + level: AccountLevel.One, + }) + mockFindUserById.mockResolvedValue({ id: userId, phone }) + mockGetIdentity.mockResolvedValue({ email: "biz@example.com" }) + mockGetUpgradeRequestList.mockResolvedValue([]) + mockCloseUpgradeRequests.mockResolvedValue(true) + }) + + it("notifies requested after the ERPNext post succeeds", async () => { + mockPostUpgradeRequest.mockResolvedValue({ name: "UPG-0001" }) + + const result = await createUpgradeRequest(accountId, input) + + expect(result).not.toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "upgrade", + phase: "requested", + status: "pending", + accountId, + meta: expect.objectContaining({ + requestId: "UPG-0001", + from: String(AccountLevel.One), + to: String(AccountLevel.Pro), + }), + }), + ) + }) + + it("does not notify when the ERPNext post fails", async () => { + mockPostUpgradeRequest.mockResolvedValue(new RepoBoomError("erpnext down")) + + const result = await createUpgradeRequest(accountId, input) + + expect(result).toBeInstanceOf(RepoBoomError) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) +}) diff --git a/test/flash/unit/app/authentication/ops-events-hooks.spec.ts b/test/flash/unit/app/authentication/ops-events-hooks.spec.ts new file mode 100644 index 000000000..b22788d5f --- /dev/null +++ b/test/flash/unit/app/authentication/ops-events-hooks.spec.ts @@ -0,0 +1,434 @@ +const mockInitiateVerify = jest.fn() +const mockIsPhoneCodeValid = jest.fn() +const mockFindUserById = jest.fn() +const mockUpdateUser = jest.fn() +const mockHasEmail = jest.fn() +const mockAddUnverifiedEmailToIdentity = jest.fn() +const mockSendEmailWithCode = jest.fn() +const mockValidateCode = jest.fn() +const mockAddPhoneToIdentity = jest.fn() +const mockGetCarrier = jest.fn() +const mockGetUserIdFromIdentifier = jest.fn() +const mockUpgradeToPhoneSchema = jest.fn() +const mockListWalletsByAccountId = jest.fn() +const mockGetBalanceForWallet = jest.fn() +const mockUpgradeAccountFromDeviceToPhone = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn().mockResolvedValue(undefined), +})) + +jest.mock("@config", () => { + const limits = { points: 100, duration: 60, blockDuration: 60 } + return { + TWILIO_ACCOUNT_SID: "AC-live", + UNSECURE_DEFAULT_LOGIN_CODE: undefined, + getGeetestConfig: jest.fn(() => ({})), + getTestAccounts: jest.fn(() => []), + getFailedLoginAttemptPerIpLimits: jest.fn(() => limits), + getFailedLoginAttemptPerLoginIdentifierLimits: jest.fn(() => limits), + getInvoiceCreateAttemptLimits: jest.fn(() => limits), + getInvoiceCreateForRecipientAttemptLimits: jest.fn(() => limits), + getOnChainAddressCreateAttemptLimits: jest.fn(() => limits), + getRequestCodePerIpLimits: jest.fn(() => limits), + getRequestCodePerLoginIdentifierLimits: jest.fn(() => limits), + getAccountsOnboardConfig: jest.fn(() => ({ + phoneMetadataValidationSettings: { enabled: false }, + ipMetadataValidationSettings: { enabled: false }, + })), + } +}) + +jest.mock("@services/tracing", () => ({ + addAttributesToCurrentSpan: jest.fn(), + recordExceptionInCurrentSpan: jest.fn(), +})) + +jest.mock("@services/ipfetcher", () => ({ + IpFetcher: jest.fn(() => ({ fetchIPInfo: jest.fn() })), +})) + +jest.mock("@app/accounts", () => ({ + upgradeAccountFromDeviceToPhone: (...args: unknown[]) => + mockUpgradeAccountFromDeviceToPhone(...args), +})) + +jest.mock("@app/accounts/create-account", () => ({ + createAccountForDeviceAccount: jest.fn(), +})) + +jest.mock("@app/wallets", () => ({ + getBalanceForWallet: (...args: unknown[]) => mockGetBalanceForWallet(...args), +})) + +jest.mock("@services/geetest", () => ({ + __esModule: true, + default: jest.fn(() => ({ validate: jest.fn() })), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})) + +jest.mock("@services/rate-limit", () => ({ + consumeLimiter: jest.fn(async () => true), +})) + +jest.mock("@app/authentication/ratelimits", () => ({ + checkFailedLoginAttemptPerIpLimits: jest.fn(async () => true), + checkFailedLoginAttemptPerLoginIdentifierLimits: jest.fn(async () => true), + rewardFailedLoginAttemptPerIpLimits: jest.fn(async () => true), + rewardFailedLoginAttemptPerLoginIdentifierLimits: jest.fn(async () => true), +})) + +jest.mock("@services/twilio", () => ({ + TWILIO_ACCOUNT_TEST: "AC-test", + TwilioClient: jest.fn(() => ({ + initiateVerify: (...args: unknown[]) => mockInitiateVerify(...args), + getCarrier: (...args: unknown[]) => mockGetCarrier(...args), + })), + isPhoneCodeValid: (...args: unknown[]) => mockIsPhoneCodeValid(...args), +})) + +jest.mock("@services/mongoose", () => ({ + UsersRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindUserById(...args), + update: (...args: unknown[]) => mockUpdateUser(...args), + })), + WalletsRepository: jest.fn(() => ({ + listByAccountId: (...args: unknown[]) => mockListWalletsByAccountId(...args), + })), +})) + +jest.mock("@services/kratos", () => { + class PhoneAccountAlreadyExistsNeedToSweepFundsError extends Error {} + class PhoneAccountAlreadyExistsCannotUpgradeError extends Error {} + return { + PhoneAccountAlreadyExistsNeedToSweepFundsError, + PhoneAccountAlreadyExistsCannotUpgradeError, + AuthWithEmailPasswordlessService: jest.fn(() => ({ + hasEmail: (...args: unknown[]) => mockHasEmail(...args), + addUnverifiedEmailToIdentity: (...args: unknown[]) => + mockAddUnverifiedEmailToIdentity(...args), + sendEmailWithCode: (...args: unknown[]) => mockSendEmailWithCode(...args), + validateCode: (...args: unknown[]) => mockValidateCode(...args), + addPhoneToIdentity: (...args: unknown[]) => mockAddPhoneToIdentity(...args), + })), + AuthWithPhonePasswordlessService: jest.fn(() => ({})), + AuthWithUsernamePasswordDeviceIdService: jest.fn(() => ({ + upgradeToPhoneSchema: (...args: unknown[]) => mockUpgradeToPhoneSchema(...args), + })), + IdentityRepository: jest.fn(() => ({ + getUserIdFromIdentifier: (...args: unknown[]) => + mockGetUserIdFromIdentifier(...args), + })), + } +}) + +import { requestPhoneCodeForAuthedUser } from "@app/authentication/request-code" +import { verifyPhone } from "@app/authentication/phone" +import { addEmailToIdentity, verifyEmail } from "@app/authentication/email" +import { loginDeviceUpgradeWithPhone } from "@app/authentication/login" +import { IdentifierNotFoundError } from "@domain/authentication/errors" +import { notifyOpsEvent } from "@services/alerts/ops-events" + +class PhoneCodeInvalidError extends Error {} +class KratosBoomError extends Error {} + +const userId = "11111111-1111-4111-8111-111111111111" as UserId +const phone = "+18765550100" as PhoneNumber +const email = "jabari@example.com" as EmailAddress +const ip = "127.0.0.1" as IpAddress + +describe("ops events — verification funnel hooks", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + describe("requestPhoneCodeForAuthedUser", () => { + const user = { id: userId, phone: undefined } as unknown as User + + it("notifies otp-sent after Twilio initiateVerify succeeds", async () => { + mockInitiateVerify.mockResolvedValue(true) + + const result = await requestPhoneCodeForAuthedUser({ + phone, + ip, + channel: "sms" as ChannelType, + user, + }) + + expect(result).toBe(true) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "otp-sent", + status: "pending", + userId, + phone, + }), + ) + }) + + it("does not notify when initiateVerify fails", async () => { + const twilioError = new Error("twilio down") + mockInitiateVerify.mockResolvedValue(twilioError) + + const result = await requestPhoneCodeForAuthedUser({ + phone, + ip, + channel: "sms" as ChannelType, + user, + }) + + expect(result).toBe(twilioError) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) + + describe("verifyPhone", () => { + it("notifies otp-verified on success", async () => { + mockIsPhoneCodeValid.mockResolvedValue(true) + const user = { id: userId, phone: undefined } + mockFindUserById.mockResolvedValue(user) + mockUpdateUser.mockResolvedValue({ ...user, phone }) + mockAddPhoneToIdentity.mockResolvedValue({}) + + const result = await verifyPhone({ userId, phone, code: "123456" as PhoneCode, ip }) + + expect(result).not.toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "otp-verified", + status: "success", + userId, + phone, + }), + ) + }) + + it("notifies otp-failed with the error name on failure", async () => { + mockIsPhoneCodeValid.mockResolvedValue(new PhoneCodeInvalidError("bad code")) + + const result = await verifyPhone({ userId, phone, code: "000000" as PhoneCode, ip }) + + expect(result).toBeInstanceOf(PhoneCodeInvalidError) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "otp-failed", + status: "failed", + userId, + phone, + error: "PhoneCodeInvalidError", + }), + ) + }) + }) + + describe("addEmailToIdentity", () => { + it("notifies otp-sent after the email code is sent", async () => { + mockHasEmail.mockResolvedValue(false) + mockAddUnverifiedEmailToIdentity.mockResolvedValue({}) + mockSendEmailWithCode.mockResolvedValue("flow-123" as EmailRegistrationId) + mockFindUserById.mockResolvedValue({ id: userId }) + + const result = await addEmailToIdentity({ email, userId }) + + expect(result).not.toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "otp-sent", + status: "pending", + userId, + email, + }), + ) + }) + + it("does not notify when sending the code fails", async () => { + mockHasEmail.mockResolvedValue(false) + mockAddUnverifiedEmailToIdentity.mockResolvedValue({}) + mockSendEmailWithCode.mockResolvedValue(new KratosBoomError("smtp down")) + + const result = await addEmailToIdentity({ email, userId }) + + expect(result).toBeInstanceOf(KratosBoomError) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) + + describe("verifyEmail", () => { + const emailRegistrationId = "flow-123" as EmailRegistrationId + + it("notifies otp-verified on success", async () => { + mockValidateCode.mockResolvedValue({ + kratosUserId: userId, + email, + totpRequired: false, + }) + mockFindUserById.mockResolvedValue({ id: userId }) + + const result = await verifyEmail({ + emailRegistrationId, + code: "123456" as EmailCode, + }) + + expect(result).not.toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "otp-verified", + status: "success", + userId, + email, + meta: { emailFlowId: emailRegistrationId }, + }), + ) + }) + + it("notifies otp-failed with the error name on failure", async () => { + mockValidateCode.mockResolvedValue(new KratosBoomError("bad code")) + + const result = await verifyEmail({ + emailRegistrationId, + code: "000000" as EmailCode, + }) + + expect(result).toBeInstanceOf(KratosBoomError) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "otp-failed", + status: "failed", + error: "KratosBoomError", + meta: { emailFlowId: emailRegistrationId }, + }), + ) + }) + }) + + describe("loginDeviceUpgradeWithPhone", () => { + const accountId = "64df1a2b3c4d5e6f78901234" as AccountId + const account = { id: accountId, kratosUserId: userId } as unknown as Account + const args = { phone, code: "123456" as PhoneCode, ip, account } + + beforeEach(() => { + mockIsPhoneCodeValid.mockResolvedValue(true) + mockGetCarrier.mockResolvedValue(new Error("carrier lookup disabled")) + mockUpgradeToPhoneSchema.mockResolvedValue(true) + mockUpgradeAccountFromDeviceToPhone.mockResolvedValue({ id: accountId }) + }) + + it("notifies upgrade-collision when the device account still has balance", async () => { + mockGetUserIdFromIdentifier.mockResolvedValue("existing-user-id") + mockListWalletsByAccountId.mockResolvedValue([ + { id: "w1", currency: "USD" }, + { id: "w2", currency: "USD" }, + ]) + mockGetBalanceForWallet + .mockResolvedValueOnce({ isZero: () => true }) + .mockResolvedValueOnce({ isZero: () => false }) + + const result = await loginDeviceUpgradeWithPhone(args) + + expect(result).toBeInstanceOf(Error) + expect((result as Error).constructor.name).toBe( + "PhoneAccountAlreadyExistsNeedToSweepFundsError", + ) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "upgrade-collision", + status: "failed", + accountId, + userId, + phone, + error: "PhoneAccountAlreadyExistsNeedToSweepFundsError", + meta: { deviceHasBalance: "true" }, + }), + ) + }) + + it("notifies upgrade-collision when the device account has zero balance", async () => { + mockGetUserIdFromIdentifier.mockResolvedValue("existing-user-id") + mockListWalletsByAccountId.mockResolvedValue([{ id: "w1", currency: "USD" }]) + mockGetBalanceForWallet.mockResolvedValue({ isZero: () => true }) + + const result = await loginDeviceUpgradeWithPhone(args) + + expect(result).toBeInstanceOf(Error) + expect((result as Error).constructor.name).toBe( + "PhoneAccountAlreadyExistsCannotUpgradeError", + ) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "upgrade-collision", + status: "failed", + accountId, + userId, + phone, + error: "PhoneAccountAlreadyExistsCannotUpgradeError", + meta: { deviceHasBalance: "false" }, + }), + ) + }) + + it("does not notify failure events on the successful upgrade path", async () => { + // The promotion event itself is emitted inside + // upgradeAccountFromDeviceToPhone (mocked here; covered by the + // accounts ops-events spec) — this function must add nothing else. + mockGetUserIdFromIdentifier.mockResolvedValue(new IdentifierNotFoundError()) + + const result = await loginDeviceUpgradeWithPhone(args) + + expect(result).toEqual({ success: true }) + expect(mockUpgradeAccountFromDeviceToPhone).toHaveBeenCalled() + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + + it("notifies upgrade-failed when the OTP is invalid", async () => { + mockIsPhoneCodeValid.mockResolvedValue(new PhoneCodeInvalidError("bad code")) + + const result = await loginDeviceUpgradeWithPhone(args) + + expect(result).toBeInstanceOf(PhoneCodeInvalidError) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "upgrade-failed", + status: "failed", + accountId, + userId, + phone, + error: "PhoneCodeInvalidError", + }), + ) + }) + + it("notifies upgrade-failed when the kratos schema upgrade fails", async () => { + mockGetUserIdFromIdentifier.mockResolvedValue(new IdentifierNotFoundError()) + mockUpgradeToPhoneSchema.mockResolvedValue(new KratosBoomError("kratos down")) + + const result = await loginDeviceUpgradeWithPhone(args) + + expect(result).toBeInstanceOf(KratosBoomError) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "verification", + phase: "upgrade-failed", + status: "failed", + error: "KratosBoomError", + }), + ) + }) + }) +}) diff --git a/test/flash/unit/app/offers/ops-events-cashout-manager.spec.ts b/test/flash/unit/app/offers/ops-events-cashout-manager.spec.ts new file mode 100644 index 000000000..9d48a8bee --- /dev/null +++ b/test/flash/unit/app/offers/ops-events-cashout-manager.spec.ts @@ -0,0 +1,185 @@ +const mockStorageGet = jest.fn() +const mockFindWalletById = jest.fn() +const mockValidOfferFrom = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn(), + toDisplayAmount: jest.requireActual("@services/alerts/ops-events").toDisplayAmount, +})) + +jest.mock("@config", () => ({ + Cashout: { OfferConfig: {}, SkipPayment: false }, + ExchangeRates: {}, +})) + +jest.mock("@app/cash-wallet-cutover/cashout-routing", () => ({ + resolveCashoutWalletSelection: jest.fn(), +})) + +jest.mock("@services/ibex/client", () => ({ + __esModule: true, + default: {}, +})) + +jest.mock("@services/ledger/caching", () => ({ + getBankOwnerIbexAccount: jest.fn(), +})) + +jest.mock("@services/email", () => ({ + EmailService: { sendCashoutInitiatedEmail: jest.fn() }, +})) + +jest.mock("@services/frappe/ErpNext", () => ({ + __esModule: true, + default: {}, +})) + +jest.mock("@services/mongoose", () => ({ + AccountsRepository: jest.fn(() => ({})), + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + })), +})) + +jest.mock("@app/offers/storage/Redis", () => ({ + __esModule: true, + default: { get: (...args: unknown[]) => mockStorageGet(...args) }, +})) + +jest.mock("@app/offers/ValidOffer", () => ({ + __esModule: true, + default: { from: (...args: unknown[]) => mockValidOfferFrom(...args) }, +})) + +import CashoutManager from "@app/offers/CashoutManager" +import { EmailService } from "@services/email" +import { USDAmount } from "@domain/shared" +import { notifyOpsEvent } from "@services/alerts/ops-events" + +const offerId = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" as OfferId +const walletId = "11111111-1111-4111-8111-111111111111" as WalletId +const accountId = "64df1a2b3c4d5e6f78901234" as AccountId + +const makeOffer = () => { + const amount = USDAmount.cents("50000") + if (amount instanceof Error) throw amount + return { + details: { + payment: { userAcct: walletId, flashAcct: "flash-wallet", amount }, + }, + } +} + +describe("ops events — CashoutManager.executeCashout", () => { + beforeEach(() => { + jest.clearAllMocks() + mockStorageGet.mockResolvedValue(makeOffer()) + mockFindWalletById.mockResolvedValue({ id: walletId, accountId }) + }) + + it("notifies initiated then succeeded on a successful cashout", async () => { + mockValidOfferFrom.mockResolvedValue({ + execute: jest.fn(async () => ({ + cashoutId: "ACC-CSH-2026-00001", + erpSubmitted: true, + })), + }) + + const result = await CashoutManager.executeCashout(offerId, walletId) + + expect(result).not.toBeInstanceOf(Error) + expect(EmailService.sendCashoutInitiatedEmail).toHaveBeenCalled() + expect(notifyOpsEvent).toHaveBeenCalledTimes(2) + expect(notifyOpsEvent).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + flow: "cashout", + phase: "initiated", + status: "pending", + accountId, + // display units, not cents: 50000 cents -> $500.00 + amount: { value: "500.00", currency: "USD" }, + meta: { offerId }, + }), + ) + expect(notifyOpsEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + flow: "cashout", + phase: "succeeded", + status: "success", + accountId, + amount: { value: "500.00", currency: "USD" }, + meta: { offerId, cashoutId: "ACC-CSH-2026-00001" }, + }), + ) + }) + + it("does not notify succeeded when the ERPNext submit partially failed", async () => { + mockValidOfferFrom.mockResolvedValue({ + execute: jest.fn(async () => ({ + cashoutId: "ACC-CSH-2026-00001", + erpSubmitted: false, + })), + }) + + const result = await CashoutManager.executeCashout(offerId, walletId) + + // caller-visible result is unchanged; the terminal failed event was + // already emitted by ValidOffer.execute + expect(result).not.toBeInstanceOf(Error) + expect(EmailService.sendCashoutInitiatedEmail).toHaveBeenCalled() + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ flow: "cashout", phase: "initiated" }), + ) + }) + + it("notifies a terminal failed event when offer validation fails", async () => { + mockValidOfferFrom.mockResolvedValue(new Error("offer no longer valid")) + + const result = await CashoutManager.executeCashout(offerId, walletId) + + expect(result).toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledTimes(2) + expect(notifyOpsEvent).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + flow: "cashout", + phase: "failed", + status: "failed", + accountId, + amount: { value: "500.00", currency: "USD" }, + step: "validation", + error: "Error", + meta: { offerId }, + }), + ) + }) + + it("notifies initiated but not succeeded when execution fails", async () => { + mockValidOfferFrom.mockResolvedValue({ + execute: jest.fn(async () => new Error("erp down")), + }) + + const result = await CashoutManager.executeCashout(offerId, walletId) + + expect(result).toBeInstanceOf(Error) + expect(EmailService.sendCashoutInitiatedEmail).not.toHaveBeenCalled() + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ flow: "cashout", phase: "initiated", status: "pending" }), + ) + }) + + it("does not notify when the caller wallet does not own the offer", async () => { + mockFindWalletById + .mockResolvedValueOnce({ id: walletId, accountId: "other-account" }) + .mockResolvedValueOnce({ id: walletId, accountId }) + + const result = await CashoutManager.executeCashout(offerId, walletId) + + expect(result).toBeInstanceOf(Error) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/app/offers/ops-events-valid-offer.spec.ts b/test/flash/unit/app/offers/ops-events-valid-offer.spec.ts new file mode 100644 index 000000000..c1f5306f5 --- /dev/null +++ b/test/flash/unit/app/offers/ops-events-valid-offer.spec.ts @@ -0,0 +1,172 @@ +const mockDraftCashout = jest.fn() +const mockSubmitCashout = jest.fn() +const mockPayInvoice = jest.fn() +const mockFindWalletById = jest.fn() +const mockFindAccountById = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn().mockResolvedValue(undefined), +})) + +jest.mock("@config", () => ({ + Cashout: { SkipPayment: false }, +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})) + +jest.mock("@services/mongoose", () => ({ + AccountsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindAccountById(...args), + })), + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + })), +})) + +jest.mock("@services/ibex/client", () => ({ + __esModule: true, + default: { payInvoice: (...args: unknown[]) => mockPayInvoice(...args) }, +})) + +jest.mock("@services/frappe/ErpNext", () => ({ + __esModule: true, + default: { + draftCashout: (...args: unknown[]) => mockDraftCashout(...args), + submitCashout: (...args: unknown[]) => mockSubmitCashout(...args), + }, +})) + +jest.mock("@app/offers/Validator", () => ({ + CashoutValidator: jest.fn(async (inputs) => inputs), +})) + +import ValidOffer, { InitiatedCashout } from "@app/offers/ValidOffer" +import { CashoutDraftError, CashoutSubmitError } from "@services/frappe/errors" +import { IbexError } from "@services/ibex/errors" +import { USDAmount } from "@domain/shared" +import { notifyOpsEvent } from "@services/alerts/ops-events" + +const walletId = "11111111-1111-4111-8111-111111111111" as WalletId +const accountId = "64df1a2b3c4d5e6f78901234" as AccountId +const cashoutId = "ACC-CSH-2026-00001" + +const makeDetails = () => { + const amount = USDAmount.cents("50000") + if (amount instanceof Error) throw amount + return { + payment: { + userAcct: walletId, + flashAcct: "22222222-2222-4222-8222-222222222222" as WalletId, + invoice: { paymentRequest: "lnbc1..." }, + amount, + }, + payout: { + bankAccountId: "bank-1", + amount, + serviceFee: amount, + }, + } as unknown as Parameters[0] +} + +const makeOffer = async (): Promise => { + const offer = await ValidOffer.from(makeDetails()) + if (offer instanceof Error) throw offer + return offer +} + +describe("ops events — ValidOffer.execute step failures", () => { + beforeEach(() => { + jest.clearAllMocks() + mockFindWalletById.mockResolvedValue({ id: walletId, accountId }) + mockFindAccountById.mockResolvedValue({ id: accountId }) + mockDraftCashout.mockResolvedValue(cashoutId) + mockPayInvoice.mockResolvedValue({ status: 2 }) + mockSubmitCashout.mockResolvedValue(true) + }) + + it("does not notify when every step succeeds", async () => { + const offer = await makeOffer() + + const result = await offer.execute() + + expect(result).toBeInstanceOf(InitiatedCashout) + expect((result as InitiatedCashout).erpSubmitted).toBe(true) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + + it("reports the draftCashout step on ERPNext draft failure", async () => { + const draftError = new CashoutDraftError("erp down") + mockDraftCashout.mockResolvedValue(draftError) + const offer = await makeOffer() + + const result = await offer.execute() + + expect(result).toBe(draftError) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "cashout", + phase: "failed", + status: "failed", + accountId, + step: "draftCashout", + error: "CashoutDraftError", + }), + ) + }) + + it("reports the payInvoice step on Ibex payment failure", async () => { + const ibexError = new IbexError(new Error("no route")) + mockPayInvoice.mockResolvedValue(ibexError) + const offer = await makeOffer() + + const result = await offer.execute() + + expect(result).toBe(ibexError) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + step: "payInvoice", + error: "IbexError", + status: "failed", + }), + ) + }) + + it("reports the submitCashout step when submit fails after the retry", async () => { + const submitError = new CashoutSubmitError("erp submit down") + mockSubmitCashout.mockResolvedValue(submitError) + const offer = await makeOffer() + + const result = await offer.execute() + + // submit failure is not surfaced as an error — manual intervention path — + // but the partial-failure fact is flagged for the caller + expect(result).toBeInstanceOf(InitiatedCashout) + expect((result as InitiatedCashout).erpSubmitted).toBe(false) + expect(mockSubmitCashout).toHaveBeenCalledTimes(2) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + step: "submitCashout", + error: "CashoutSubmitError", + status: "failed", + }), + ) + }) + + it("does not report submitCashout when the retry succeeds", async () => { + mockSubmitCashout + .mockResolvedValueOnce(new CashoutSubmitError("transient")) + .mockResolvedValueOnce(true) + const offer = await makeOffer() + + const result = await offer.execute() + + expect(result).toBeInstanceOf(InitiatedCashout) + expect((result as InitiatedCashout).erpSubmitted).toBe(true) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/app/payments/send-intraledger.spec.ts b/test/flash/unit/app/payments/send-intraledger.spec.ts index f92548a46..7f26be2fd 100644 --- a/test/flash/unit/app/payments/send-intraledger.spec.ts +++ b/test/flash/unit/app/payments/send-intraledger.spec.ts @@ -13,6 +13,10 @@ jest.mock("@services/tracing", () => ({ recordExceptionInCurrentSpan: jest.fn(), })) +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn().mockResolvedValue(undefined), +})) + jest.mock("@app/prices", () => ({ btcFromUsdMidPriceFn: jest.fn(), getCurrentPriceAsDisplayPriceRatio: jest.fn(), @@ -91,6 +95,7 @@ jest.mock("@app/payments/helpers", () => ({ import { intraledgerPaymentSendWalletIdForUsdWallet } from "@app/payments/send-intraledger" import { MismatchedCurrencyForWalletError } from "@domain/errors" import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" +import { notifyOpsEvent } from "@services/alerts/ops-events" const senderUsdWalletId = "11111111-1111-4111-8111-111111111111" as WalletId const senderUsdtWalletId = "22222222-2222-4222-8222-222222222222" as WalletId @@ -260,3 +265,129 @@ describe("intraledgerPaymentSendWalletIdForUsdWallet", () => { expect(mockPayInvoice).not.toHaveBeenCalled() }) }) + +describe("intraledger send ops events", () => { + beforeEach(() => { + jest.clearAllMocks() + + mockFindAccountById.mockImplementation(async (accountId: AccountId) => + activeAccount(accountId as string), + ) + mockAddInvoice.mockResolvedValue({ invoice: { bolt11: "lnbc1recipient" } }) + mockPayInvoice.mockResolvedValue({ status: 2 }) + mockFindWalletById.mockImplementation(async (walletId: WalletId) => + wallet({ + id: walletId, + accountId: + walletId === senderUsdWalletId ? "sender-account" : "recipient-account", + currency: WalletCurrency.Usd, + }), + ) + }) + + const sendArgs = { + senderWalletId: senderUsdWalletId, + recipientWalletId: recipientUsdWalletId, + amount: 100, + memo: "ops event test", + } + + it("notifies a succeeded transfer event with display amount on success", async () => { + const result = await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(result).toEqual({ value: "success" }) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "succeeded", + status: "success", + // display units, not cents: 100 cents -> $1.00 + amount: { value: "1.00", currency: "USD" }, + meta: { + senderWalletId: senderUsdWalletId, + recipientWalletId: recipientUsdWalletId, + }, + }), + ) + }) + + it("notifies a failed transfer event with the error name on error return", async () => { + mockFindWalletById.mockImplementation(async (walletId: WalletId) => + wallet({ + id: walletId, + accountId: + walletId === senderUsdWalletId ? "sender-account" : "recipient-account", + currency: + walletId === senderUsdWalletId ? WalletCurrency.Usd : WalletCurrency.Usdt, + }), + ) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(result).toBeInstanceOf(MismatchedCurrencyForWalletError) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "failed", + status: "failed", + error: "MismatchedCurrencyForWalletError", + meta: expect.objectContaining({ reason: "error-return" }), + }), + ) + }) + + it("notifies a failed transfer event when the sender wallet fails wrapper validation", async () => { + mockFindWalletById.mockImplementation(async (walletId: WalletId) => + wallet({ + id: walletId, + accountId: + walletId === senderUsdWalletId ? "sender-account" : "recipient-account", + currency: WalletCurrency.Btc, + }), + ) + + const result = await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(result).toBeInstanceOf(MismatchedCurrencyForWalletError) + expect(mockAddInvoice).not.toHaveBeenCalled() + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "failed", + status: "failed", + error: "MismatchedCurrencyForWalletError", + amount: { value: "1.00", currency: "USD" }, + meta: expect.objectContaining({ reason: "error-return" }), + }), + ) + }) + + it("distinguishes an Ibex status failure from an error return", async () => { + mockPayInvoice.mockResolvedValue({ status: 3 }) + + await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "failed", + status: "failed", + meta: expect.objectContaining({ reason: "status-failure" }), + }), + ) + expect((notifyOpsEvent as jest.Mock).mock.calls[0][0].error).toBeUndefined() + }) + + it("notifies a pending transfer event when Ibex reports pending", async () => { + mockPayInvoice.mockResolvedValue({ status: 1 }) + + await intraledgerPaymentSendWalletIdForUsdWallet(sendArgs) + + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ flow: "transfer", phase: "pending", status: "pending" }), + ) + }) +}) diff --git a/test/flash/unit/app/payments/send-lightning-ops-events.spec.ts b/test/flash/unit/app/payments/send-lightning-ops-events.spec.ts new file mode 100644 index 000000000..c715fbd4d --- /dev/null +++ b/test/flash/unit/app/payments/send-lightning-ops-events.spec.ts @@ -0,0 +1,218 @@ +const mockFindWalletById = jest.fn() +const mockDecodeInvoice = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn().mockResolvedValue(undefined), +})) + +jest.mock("@config", () => ({ + getCallbackServiceConfig: jest.fn(() => ({})), +})) + +jest.mock("@services/tracing", () => ({ + addAttributesToCurrentSpan: jest.fn(), + recordExceptionInCurrentSpan: jest.fn(), +})) + +jest.mock("@services/lnd", () => ({ + LndService: jest.fn(() => ({})), +})) + +jest.mock("@services/mongoose", () => ({ + LnPaymentsRepository: jest.fn(() => ({})), + PaymentFlowStateRepository: jest.fn(() => ({})), + WalletInvoicesRepository: jest.fn(() => ({})), + WalletsRepository: jest.fn(() => ({ + findById: (...args: unknown[]) => mockFindWalletById(...args), + })), + UsersRepository: jest.fn(() => ({})), + AccountsRepository: jest.fn(() => ({})), +})) + +jest.mock("@services/dealer-price", () => ({ + DealerPriceService: jest.fn(() => ({})), +})) + +jest.mock("@services/ledger", () => ({ + LedgerService: jest.fn(() => ({})), +})) + +jest.mock("@services/ledger/facade", () => ({})) + +jest.mock("@services/lock", () => ({ + LockService: jest.fn(() => ({})), +})) + +jest.mock("@services/notifications", () => ({ + NotificationsService: jest.fn(() => ({})), +})) + +jest.mock("@services/svix", () => ({ + CallbackService: jest.fn(() => ({})), +})) + +jest.mock("@app/prices", () => ({ + getCurrentPriceAsDisplayPriceRatio: jest.fn(), +})) + +jest.mock("@app/users/remove-device-tokens", () => ({ + removeDeviceTokens: jest.fn(), +})) + +jest.mock("@app/wallets", () => ({ + validateIsBtcWallet: jest.fn(), + validateIsUsdWallet: jest.fn(), +})) + +jest.mock("@app/payments/helpers", () => ({ + addContactsAfterSend: jest.fn(), + checkIntraledgerLimits: jest.fn(), + checkTradeIntraAccountLimits: jest.fn(), + checkWithdrawalLimits: jest.fn(), + constructPaymentFlowBuilder: jest.fn(), + getPriceRatioForLimits: jest.fn(), +})) + +jest.mock("@app/payments/reimburse-fee", () => ({ + reimburseFee: jest.fn(), +})) + +jest.mock("@domain/bitcoin/lightning", () => { + const actual = jest.requireActual("@domain/bitcoin/lightning") + return { + ...actual, + decodeInvoice: (...args: unknown[]) => mockDecodeInvoice(...args), + } +}) + +import { + payInvoiceByWalletId, + payNoAmountInvoiceByWalletIdForBtcWallet, + payNoAmountInvoiceByWalletIdForUsdWallet, +} from "@app/payments/send-lightning" +import { PaymentSendStatus } from "@domain/bitcoin/lightning" +import { AlreadyPaidError, MismatchedCurrencyForWalletError } from "@domain/errors" +import { validateIsBtcWallet } from "@app/wallets" +import { notifyOpsEvent } from "@services/alerts/ops-events" + +const decodedInvoice = (amountMsat: bigint) => ({ + destination: "dest-pubkey", + paymentHash: "a".repeat(64), + description: "test invoice", + expiresAt: new Date(Date.now() + 600_000), + paymentAmount: amountMsat > 0n ? { amount: amountMsat, currency: "BTC" } : undefined, +}) + +const senderWalletId = "11111111-1111-4111-8111-111111111111" as WalletId +const invalidWalletId = "not-a-wallet-id" as WalletId +const senderAccount = { + id: "64df1a2b3c4d5e6f78901234", + displayCurrency: "USD", +} as unknown as Account + +describe("ops events — payInvoiceByWalletId", () => { + beforeEach(() => { + jest.clearAllMocks() + mockDecodeInvoice.mockReturnValue(decodedInvoice(1000n)) + }) + + it("notifies a succeeded transfer event on a non-error outcome", async () => { + // AlreadyPaid resolves to PaymentSendStatus.AlreadyPaid — a successful outcome + mockFindWalletById.mockResolvedValue(new AlreadyPaidError()) + + const result = await payInvoiceByWalletId({ + uncheckedPaymentRequest: "lnbc1...", + memo: null, + senderWalletId, + senderAccount, + }) + + expect(result).toBe(PaymentSendStatus.AlreadyPaid) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "succeeded", + status: "success", + accountId: senderAccount.id, + meta: { senderWalletId }, + }), + ) + }) + + it("notifies a succeeded transfer with display amount for a USD no-amount send", async () => { + mockDecodeInvoice.mockReturnValue(decodedInvoice(0n)) + mockFindWalletById.mockResolvedValue(new AlreadyPaidError()) + + const result = await payNoAmountInvoiceByWalletIdForUsdWallet({ + uncheckedPaymentRequest: "lnbc1...", + amount: 15000, + memo: null, + senderWalletId, + senderAccount, + }) + + expect(result).toBe(PaymentSendStatus.AlreadyPaid) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "succeeded", + status: "success", + accountId: senderAccount.id, + // display units, not cents: 15000 cents -> $150.00 + amount: { value: "150.00", currency: "USD" }, + meta: { senderWalletId }, + }), + ) + }) + + it("notifies a failed transfer with sats amount for a BTC no-amount send", async () => { + ;(validateIsBtcWallet as jest.Mock).mockResolvedValue( + new MismatchedCurrencyForWalletError(), + ) + + const result = await payNoAmountInvoiceByWalletIdForBtcWallet({ + uncheckedPaymentRequest: "lnbc1...", + amount: 2100, + memo: null, + senderWalletId, + senderAccount, + }) + + expect(result).toBeInstanceOf(MismatchedCurrencyForWalletError) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "failed", + status: "failed", + error: "MismatchedCurrencyForWalletError", + amount: { value: "2100", currency: "sats" }, + meta: { senderWalletId, reason: "error-return" }, + }), + ) + }) + + it("notifies a failed transfer event with the error name on error return", async () => { + const result = await payInvoiceByWalletId({ + uncheckedPaymentRequest: "lnbc1...", + memo: null, + senderWalletId: invalidWalletId, + senderAccount, + }) + + expect(result).toBeInstanceOf(Error) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "failed", + status: "failed", + accountId: senderAccount.id, + error: (result as Error).constructor.name, + meta: { senderWalletId: invalidWalletId, reason: "error-return" }, + }), + ) + }) +}) diff --git a/test/flash/unit/services/alerts/ops-events.spec.ts b/test/flash/unit/services/alerts/ops-events.spec.ts new file mode 100644 index 000000000..e6e894dec --- /dev/null +++ b/test/flash/unit/services/alerts/ops-events.spec.ts @@ -0,0 +1,267 @@ +let mockWebhookUrl: string | undefined = "https://discord.test/api/webhooks/ops" + +jest.mock("@config", () => ({ + get OPS_DISCORD_WEBHOOK_URL() { + return mockWebhookUrl + }, + NETWORK: "regtest", +})) + +jest.mock("@services/tracing", () => ({ + recordExceptionInCurrentSpan: jest.fn(), +})) + +jest.mock("axios", () => ({ + post: jest.fn(), + isAxiosError: (error: unknown) => + Boolean((error as { isAxiosError?: boolean })?.isAxiosError), +})) + +import axios from "axios" + +import { USDAmount, USDTAmount } from "@domain/shared" + +import { + buildEmbed, + maskEmail, + maskPhone, + notifyOpsEvent, + opsEventsSettled, + toDisplayAmount, + truncateId, + OpsEvent, +} from "@services/alerts/ops-events" + +const sendAndSettle = async (event: OpsEvent) => { + notifyOpsEvent(event) + await opsEventsSettled() +} + +const mockPost = axios.post as jest.Mock + +const baseEvent: OpsEvent = { + flow: "verification", + phase: "otp-sent", + status: "pending", +} + +const rateLimit429 = (retryAfterSecs: number) => ({ + isAxiosError: true, + response: { status: 429, data: { retry_after: retryAfterSecs }, headers: {} }, +}) + +describe("maskPhone", () => { + it("keeps leading +, first 4 digits and last 2", () => { + expect(maskPhone("+18765550100")).toBe("+1876…00") + }) + + it("works without a leading +", () => { + expect(maskPhone("18765550100")).toBe("1876…00") + }) + + it("strips formatting characters before masking", () => { + expect(maskPhone("+1 (876) 555-0100")).toBe("+1876…00") + }) + + it("does not reconstruct short numbers", () => { + expect(maskPhone("+123456")).toBe("+12…") + expect(maskPhone("12")).toBe("12…") + }) + + it("handles empty/garbage input", () => { + expect(maskPhone("")).toBe("…") + expect(maskPhone("+")).toBe("+…") + }) +}) + +describe("maskEmail", () => { + it("keeps first char and full domain", () => { + expect(maskEmail("jabari@gmail.com")).toBe("j***@gmail.com") + }) + + it("handles single-char local part", () => { + expect(maskEmail("j@x.io")).toBe("j***@x.io") + }) + + it("masks fully when not an email", () => { + expect(maskEmail("not-an-email")).toBe("***") + expect(maskEmail("@no-local-part.com")).toBe("***") + }) +}) + +describe("truncateId", () => { + it("truncates long ids to first 8 chars plus ellipsis", () => { + expect(truncateId("64df1a2b3c4d5e6f78901234")).toBe("64df1a2b…") + }) + + it("keeps short ids as-is", () => { + expect(truncateId("tr_12345")).toBe("tr_12345") + expect(truncateId("123456789012")).toBe("123456789012") + }) +}) + +describe("toDisplayAmount", () => { + it("renders USD cents as dollars", () => { + const amount = USDAmount.cents("9540") + if (amount instanceof Error) throw amount + expect(toDisplayAmount(amount)).toEqual({ value: "95.40", currency: "USD" }) + }) + + it("renders USDT micros as major units", () => { + const amount = USDTAmount.usdCents("9540") + if (amount instanceof Error) throw amount + expect(toDisplayAmount(amount)).toEqual({ value: "95.40", currency: "USDT" }) + }) +}) + +describe("buildEmbed", () => { + it("titles the embed from flow and phase, uppercasing OTP", () => { + const embed = buildEmbed({ ...baseEvent, phase: "otp-verified" }) + expect(embed.title).toBe("📲 Verification — OTP verified") + }) + + it.each([ + ["success" as const, 0x2ecc71], + ["pending" as const, 0xf39c12], + ["failed" as const, 0xe74c3c], + ])("colors %s embeds correctly", (status, color) => { + expect(buildEmbed({ ...baseEvent, status }).color).toBe(color) + }) + + it("masks identity fields and truncates ids", () => { + const embed = buildEmbed({ + flow: "cashout", + phase: "failed", + status: "failed", + accountId: "64df1a2b3c4d5e6f78901234", + phone: "+18765550100", + email: "jabari@gmail.com", + step: "payInvoice", + error: "IbexError", + meta: { offerId: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" }, + }) + + expect(embed.title).toBe("💸 Cashout — failed") + const byName = Object.fromEntries(embed.fields.map((f) => [f.name, f.value])) + expect(byName.account).toBe("64df1a2b…") + expect(byName.phone).toBe("+1876…00") + expect(byName.email).toBe("j***@gmail.com") + expect(byName.step).toBe("payInvoice") + expect(byName.error).toBe("IbexError") + expect(byName.offerId).toBe("aaaaaaaa…") + expect(byName.env).toBe("regtest") + expect(embed.timestamp).toEqual(expect.any(String)) + + const rendered = JSON.stringify(embed) + expect(rendered).not.toContain("18765550100") + expect(rendered).not.toContain("jabari@") + }) + + it("renders amounts with currency", () => { + const embed = buildEmbed({ + ...baseEvent, + amount: { value: 500n, currency: "USD" }, + }) + const amount = embed.fields.find((f) => f.name === "amount") + expect(amount?.value).toBe("500 USD") + }) +}) + +describe("notifyOpsEvent", () => { + beforeEach(() => { + jest.clearAllMocks() + mockWebhookUrl = "https://discord.test/api/webhooks/ops" + mockPost.mockResolvedValue({ status: 204 }) + }) + + it("no-ops when OPS_DISCORD_WEBHOOK_URL is unset", async () => { + mockWebhookUrl = undefined + await sendAndSettle(baseEvent) + expect(mockPost).not.toHaveBeenCalled() + }) + + it("posts a single embed to the webhook", async () => { + await sendAndSettle(baseEvent) + + expect(mockPost).toHaveBeenCalledTimes(1) + const [url, body, opts] = mockPost.mock.calls[0] + expect(url).toBe("https://discord.test/api/webhooks/ops") + expect(body.embeds).toHaveLength(1) + expect(body.embeds[0].title).toBe("📲 Verification — OTP sent") + expect(opts.timeout).toBe(3000) + }) + + it("never throws or rejects even when the post fails", async () => { + mockPost.mockRejectedValue(new Error("boom")) + expect(() => notifyOpsEvent(baseEvent)).not.toThrow() + await expect(opsEventsSettled()).resolves.toBeUndefined() + }) + + it("sends events sequentially in FIFO order", async () => { + notifyOpsEvent({ ...baseEvent, phase: "first" }) + notifyOpsEvent({ ...baseEvent, phase: "second" }) + await opsEventsSettled() + + expect(mockPost).toHaveBeenCalledTimes(2) + expect(mockPost.mock.calls[0][1].embeds[0].title).toContain("first") + expect(mockPost.mock.calls[1][1].embeds[0].title).toContain("second") + }) + + it("retries once after a 429, honoring retry_after", async () => { + mockPost + .mockRejectedValueOnce(rateLimit429(0.01)) + .mockResolvedValueOnce({ status: 204 }) + + const started = Date.now() + await sendAndSettle(baseEvent) + + expect(mockPost).toHaveBeenCalledTimes(2) + expect(Date.now() - started).toBeGreaterThanOrEqual(9) + expect(mockPost.mock.calls[1][1]).toEqual(mockPost.mock.calls[0][1]) + }) + + it("gives up after the 429 retry also fails", async () => { + mockPost + .mockRejectedValueOnce(rateLimit429(0.01)) + .mockRejectedValueOnce(rateLimit429(0.01)) + await sendAndSettle(baseEvent) + expect(mockPost).toHaveBeenCalledTimes(2) + }) + + it("does not retry non-429 failures", async () => { + mockPost.mockRejectedValue(new Error("boom")) + await sendAndSettle(baseEvent) + expect(mockPost).toHaveBeenCalledTimes(1) + }) + + it("drops oldest events past the queue cap and emits one drop summary", async () => { + let releaseFirst: (() => void) | undefined + mockPost.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseFirst = resolve + }), + ) + mockPost.mockResolvedValue({ status: 204 }) + + const total = 60 + for (let i = 0; i < total; i++) { + notifyOpsEvent({ ...baseEvent, phase: `event-${i}` }) + } + // First post is in flight; 59 queued behind it; 9 oldest dropped past the cap. + releaseFirst?.() + await opsEventsSettled() + + // 1 in-flight + 50 queued survivors + 1 summary embed + expect(mockPost).toHaveBeenCalledTimes(52) + const lastBody = mockPost.mock.calls[51][1] + expect(lastBody.embeds[0].title).toContain("9 events dropped") + // Oldest queued events were the ones dropped + const sentTitles = mockPost.mock.calls + .slice(0, 51) + .map((call) => call[1].embeds[0].title) + expect(sentTitles[0]).toContain("event 0") + expect(sentTitles[1]).toContain("event 10") + expect(sentTitles[50]).toContain("event 59") + }) +}) diff --git a/test/flash/unit/services/bridge/webhook-server/ops-events-webhooks.spec.ts b/test/flash/unit/services/bridge/webhook-server/ops-events-webhooks.spec.ts new file mode 100644 index 000000000..74dfe50f7 --- /dev/null +++ b/test/flash/unit/services/bridge/webhook-server/ops-events-webhooks.spec.ts @@ -0,0 +1,241 @@ +const mockCreateBridgeDeposit = jest.fn() +const mockWriteDepositRequest = jest.fn() +const mockLockIdempotencyKey = jest.fn() +const mockUpdateWithdrawalStatus = jest.fn() +const mockWriteCashoutCompleted = jest.fn() +const mockWriteCashoutFailed = jest.fn() + +jest.mock("@services/alerts/ops-events", () => ({ + notifyOpsEvent: jest.fn().mockResolvedValue(undefined), +})) + +jest.mock("@services/alerts", () => ({ + alertBridge: jest.fn(), + generateDedupKey: new Proxy({}, { get: () => jest.fn(() => "dedup") }), +})) + +jest.mock("@services/alerts/ibex-bridge-movement", () => ({ + alertIbexReconciliationFailed: jest.fn(), +})) + +jest.mock("@services/lock", () => ({ + LockService: jest.fn(() => ({ + lockIdempotencyKey: (...args: unknown[]) => mockLockIdempotencyKey(...args), + })), +})) + +jest.mock("@services/logger", () => ({ + baseLogger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})) + +jest.mock("@services/mongoose/bridge-deposit-log", () => ({ + createBridgeDeposit: (...args: unknown[]) => mockCreateBridgeDeposit(...args), +})) + +jest.mock("@services/bridge/reconciliation", () => ({ + reconcileByTxHash: jest.fn().mockResolvedValue({ status: "matched" }), +})) + +jest.mock("@services/frappe/BridgeTransferRequestWriter", () => ({ + writeBridgeDepositRequest: (...args: unknown[]) => mockWriteDepositRequest(...args), + writeBridgeCashoutCompleted: (...args: unknown[]) => mockWriteCashoutCompleted(...args), + writeBridgeCashoutFailed: (...args: unknown[]) => mockWriteCashoutFailed(...args), +})) + +jest.mock("@services/mongoose/bridge-accounts", () => ({ + BRIDGE_WITHDRAWAL_NOT_FOUND: "Bridge withdrawal not found", + updateWithdrawalStatus: (...args: unknown[]) => mockUpdateWithdrawalStatus(...args), +})) + +jest.mock("@app/bridge/send-withdrawal-notification", () => ({ + sendBridgeWithdrawalNotificationBestEffort: jest.fn().mockResolvedValue(undefined), +})) + +import { Request, Response } from "express" +import { depositHandler } from "@services/bridge/webhook-server/routes/deposit" +import { transferHandler } from "@services/bridge/webhook-server/routes/transfer" +import { notifyOpsEvent } from "@services/alerts/ops-events" + +const makeRes = () => { + const res = { status: jest.fn(), json: jest.fn() } as unknown as Response + ;(res.status as jest.Mock).mockReturnValue(res) + ;(res.json as jest.Mock).mockReturnValue(res) + return res +} + +const makeReq = (body: Record) => ({ body }) as unknown as Request + +describe("ops events — bridge webhook hooks", () => { + beforeEach(() => { + jest.clearAllMocks() + mockLockIdempotencyKey.mockResolvedValue(true) + }) + + describe("depositHandler", () => { + const depositBody = { + event_id: "wh_123", + event_category: "transfer", + event_object: { + id: "tr_xyz789abc123", + state: "payment_processed", + amount: "25.00", + currency: "usd", + on_behalf_of: "cust_bob_1234567890", + }, + } + + it("notifies deposit succeeded when processing completes", async () => { + mockCreateBridgeDeposit.mockResolvedValue({}) + mockWriteDepositRequest.mockResolvedValue({}) + const res = makeRes() + + await depositHandler(makeReq(depositBody), res) + + expect(res.status).toHaveBeenCalledWith(200) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "deposit", + phase: "succeeded", + status: "success", + amount: { value: "25.00", currency: "usd" }, + meta: expect.objectContaining({ transferId: "tr_xyz789abc123" }), + }), + ) + }) + + it("notifies deposit failed when persistence fails", async () => { + mockCreateBridgeDeposit.mockResolvedValue(new Error("mongo down")) + const res = makeRes() + + await depositHandler(makeReq(depositBody), res) + + expect(res.status).toHaveBeenCalledWith(500) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "deposit", + phase: "failed", + status: "failed", + step: "persist-deposit-log", + error: "Error", + }), + ) + }) + + it("notifies deposit failed when the ERPNext audit write fails", async () => { + mockCreateBridgeDeposit.mockResolvedValue({}) + mockWriteDepositRequest.mockResolvedValue(new Error("erpnext down")) + const res = makeRes() + + await depositHandler(makeReq(depositBody), res) + + expect(res.status).toHaveBeenCalledWith(500) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "deposit", + phase: "failed", + status: "failed", + step: "erpnext-audit", + }), + ) + }) + + it("does not notify for duplicate deliveries", async () => { + mockCreateBridgeDeposit.mockResolvedValue({}) + mockWriteDepositRequest.mockResolvedValue({}) + mockLockIdempotencyKey.mockResolvedValue(new Error("already locked")) + const res = makeRes() + + await depositHandler(makeReq(depositBody), res) + + expect(res.status).toHaveBeenCalledWith(200) + expect(res.json).toHaveBeenCalledWith({ status: "already_processed" }) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) + + describe("transferHandler", () => { + const transferBody = ( + state: string, + event = "transfer.updated.status_transitioned", + ) => ({ + event_id: "wh_456", + event_type: event, + event_object: { id: "tr_transfer_123", state, amount: "10.00", currency: "usd" }, + }) + + it("notifies transfer succeeded on completion", async () => { + mockUpdateWithdrawalStatus.mockResolvedValue({ + accountId: "64df1a2b3c4d5e6f78901234", + amount: "10.00", + currency: "usd", + }) + mockWriteCashoutCompleted.mockResolvedValue({}) + const res = makeRes() + + await transferHandler(makeReq(transferBody("payment_processed")), res) + + expect(res.status).toHaveBeenCalledWith(200) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "succeeded", + status: "success", + accountId: "64df1a2b3c4d5e6f78901234", + amount: { value: "10.00", currency: "usd" }, + meta: { transferId: "tr_transfer_123" }, + }), + ) + }) + + it("notifies transfer failed on a terminal failure state", async () => { + mockUpdateWithdrawalStatus.mockResolvedValue({ + accountId: "64df1a2b3c4d5e6f78901234", + amount: "10.00", + currency: "usd", + failureReason: "account_closed", + }) + mockWriteCashoutFailed.mockResolvedValue({}) + const res = makeRes() + + await transferHandler(makeReq(transferBody("returned")), res) + + expect(res.status).toHaveBeenCalledWith(200) + expect(notifyOpsEvent).toHaveBeenCalledTimes(1) + expect(notifyOpsEvent).toHaveBeenCalledWith( + expect.objectContaining({ + flow: "transfer", + phase: "failed", + status: "failed", + accountId: "64df1a2b3c4d5e6f78901234", + error: "account_closed", + }), + ) + }) + + it("does not notify for ignored transient states", async () => { + const res = makeRes() + + await transferHandler(makeReq(transferBody("refund_in_flight")), res) + + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + + it("does not notify duplicate deliveries", async () => { + mockUpdateWithdrawalStatus.mockResolvedValue({ + accountId: "64df1a2b3c4d5e6f78901234", + amount: "10.00", + currency: "usd", + }) + mockWriteCashoutCompleted.mockResolvedValue({}) + mockLockIdempotencyKey.mockResolvedValue(new Error("already locked")) + const res = makeRes() + + await transferHandler(makeReq(transferBody("payment_processed")), res) + + expect(res.json).toHaveBeenCalledWith({ status: "already_processed" }) + expect(notifyOpsEvent).not.toHaveBeenCalled() + }) + }) +})