From 1b192ca9af057e7fcc0efb794f1355f6d4c2f65a Mon Sep 17 00:00:00 2001 From: Vandana Date: Sun, 31 May 2026 08:48:18 -0700 Subject: [PATCH 1/5] feat(cutover): add operator dashboard and provisioning tools --- dev/apollo-federation/supergraph.graphql | 1 + .../Flash GraphQL API/environments/local.bru | 8 +- ...-05-26-local-cutover-operator-dashboard.md | 88 ++ ...6-05-28-cutover-dashboard-lazy-balances.md | 77 ++ src/app/cash-wallet-cutover/index.ts | 1 + .../cash-wallet-cutover/operator-dashboard.ts | 930 +++++++++++++++++ src/app/cash-wallet-cutover/orchestrator.ts | 3 + .../provision-usdt-wallets.ts | 143 +++ src/app/cash-wallet-cutover/runner.ts | 15 +- .../cash-wallet-cutover/runtime-services.ts | 24 +- src/app/cash-wallet-cutover/worker.ts | 12 +- src/scripts/cash-wallet-cutover-dashboard.ts | 973 ++++++++++++++++++ src/scripts/cash-wallet-cutover.ts | 32 + .../operator-dashboard.spec.ts | 782 ++++++++++++++ .../provision-usdt-wallets.spec.ts | 169 +++ .../app/cash-wallet-cutover/runner.spec.ts | 37 + .../runtime-services.spec.ts | 27 + .../app/cash-wallet-cutover/worker.spec.ts | 4 +- .../app/wallets/usd-wallet-amount.spec.ts | 9 + 19 files changed, 3316 insertions(+), 19 deletions(-) create mode 100644 docs/plans/2026-05-26-local-cutover-operator-dashboard.md create mode 100644 docs/plans/2026-05-28-cutover-dashboard-lazy-balances.md create mode 100644 src/app/cash-wallet-cutover/operator-dashboard.ts create mode 100644 src/app/cash-wallet-cutover/provision-usdt-wallets.ts create mode 100644 src/scripts/cash-wallet-cutover-dashboard.ts create mode 100644 test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts create mode 100644 test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts diff --git a/dev/apollo-federation/supergraph.graphql b/dev/apollo-federation/supergraph.graphql index baeee5a52..09c7b3662 100644 --- a/dev/apollo-federation/supergraph.graphql +++ b/dev/apollo-federation/supergraph.graphql @@ -343,6 +343,7 @@ type BridgeWithdrawal amount: String! createdAt: String! currency: String! + failureReason: String id: ID! status: String! } diff --git a/dev/bruno/Flash GraphQL API/environments/local.bru b/dev/bruno/Flash GraphQL API/environments/local.bru index 1cda70a56..1d803e8a3 100644 --- a/dev/bruno/Flash GraphQL API/environments/local.bru +++ b/dev/bruno/Flash GraphQL API/environments/local.bru @@ -1,13 +1,15 @@ vars { flashGraphqlUrl: http://localhost:4002/graphql admin_url: http://localhost:4001/graphql - admin_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VySWQiOiJhZG1pbiIsInJvbGVzIjpbIkFjY291bnRzIE1hbmFnZXIiXX0.UOmQR2K6RdS1FVvQbjvSQfoQ-VsTC6Y7x2YAXZImdsA currency: BTC - phone: +16505554322 + phone: +16505554320 code: 000000 - token: walletId: walletIdUsd: c593736e-5a58-42e4-93fa-dc895856c1f1 userEmail: mauriente@gmail.com userFullName: maurientes } +vars:secret [ + admin_token, + token +] diff --git a/docs/plans/2026-05-26-local-cutover-operator-dashboard.md b/docs/plans/2026-05-26-local-cutover-operator-dashboard.md new file mode 100644 index 000000000..f3a40d326 --- /dev/null +++ b/docs/plans/2026-05-26-local-cutover-operator-dashboard.md @@ -0,0 +1,88 @@ +# Local Cash Wallet Cutover Operator Dashboard Plan + +## Goal + +Build a local-only dashboard at `http://localhost:3450` that lets an operator monitor the 60 cutover test accounts and their cash wallets through each cutover state. The dashboard must use raw backend repositories and wallet balances, not the GraphQL presentation layer, because public wallet queries intentionally hide either USD or USDT depending on client capability and cutover state. + +## Constraints + +- Read-only: the dashboard must not mutate accounts, wallets, balances, migrations, or cutover config. +- Local-only: bind to localhost and serve a static browser UI plus a JSON snapshot endpoint. +- Source of truth: + - account manifests from `/tmp/eng345usd-20260526115410-local-backend-accounts.json` and `/tmp/eng345usdonly-20260526195758-accounts.json` + - raw Mongo repositories for accounts, wallets, and cash-wallet-cutover state + - `Wallets.getBalanceForWallet` for live balances +- Expected population: + - 60 accounts + - 110 current wallets before `provision-usdt-wallets` + - 120 target wallets after every USD-only account receives USDT +- No production API behavior should change. + +## Design + +1. Add a pure dashboard snapshot builder under `src/app/cash-wallet-cutover/operator-dashboard.ts`. + - Load account IDs from manifest records. + - Fetch each account by id and all raw wallets by account id. + - Identify checking USD and checking USDT wallets directly from raw wallet records. + - Fetch live balances for each cash wallet. + - Fetch cutover config and per-account migration records when config has `runId`. + - Derive summary totals and account-level anomaly badges. + +2. Add focused unit tests under `test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts`. + - Verify wallet grouping and current/target wallet counts. + - Verify missing-USDT detection for USD-only accounts. + - Verify funded USD-only count from USD cent balances. + - Verify migration status counts and anomaly flags. + +3. Add a thin local HTTP script under `src/scripts/cash-wallet-cutover-dashboard.ts`. + - Accept `--port`, `--configPath`, optional `--run-id`, optional `--cutover-version`, optional `--expected-accounts`, and repeated `--manifest` arguments. + - Default port: `3450`. + - Bind explicitly to `127.0.0.1`. + - Default manifests: + - `/tmp/eng345usd-20260526115410-local-backend-accounts.json` + - `/tmp/eng345usdonly-20260526195758-accounts.json` + - Routes: + - `GET /` static dashboard HTML/CSS/JS + - `GET /api/snapshot` live JSON snapshot + - Poll snapshot every 10 seconds from the browser, with a manual refresh button. + - Cache server-side snapshots for a short TTL so browser refreshes do not hammer IBEX. + +4. UI content. + - Compact operational layout. + - Summary strip for cutover state, run id/version, accounts, wallets current/target, missing USDT, funded USD-only, USD total, USDT total, and anomalies. + - Filters for anomalies, funded only, missing USDT, nonzero USD, nonzero USDT, and migration status. + - Per-account table with phone, account id, default wallet, USD wallet/balance, USDT wallet/balance, migration status, and anomaly badges. + - Color coding: + - green expected + - yellow pending/missing-but-expected + - red broken or dangerous anomalies + +5. Verification. + - Run the new unit test first and confirm it fails before implementation. + - Implement the snapshot builder and dashboard script. + - Run the focused unit test. + - Run TypeScript check for touched files through the repo build/test path where practical. + - Start the dashboard on `localhost:3450` and verify: + - `GET /` returns HTML + - `GET /api/snapshot` returns JSON + - dashboard process is listening on port `3450` + +## Risks + +- `Wallets.getBalanceForWallet` returns currency-specific amount shapes; the snapshot builder must normalize cautiously and preserve raw balance display for unknown shapes. +- Account manifest shape may differ between the 50-account and 10-account batches. The loader should accept common `accountId`, `account.id`, `id`, `phone`, and `username` fields and fail with clear errors if no account id can be found. +- Prepared migration records may exist before cutover config has `runId`. The dashboard must accept explicit `--run-id` and `--cutover-version` and use them for migration lookup when provided. +- The manifest loader must support the actual top-level `accounts` and `created` arrays, reject duplicate account IDs, and by default validate that 60 accounts were loaded. +- Balance reads can be expensive across 110-120 wallets. The local server must cache snapshots with a short TTL, capture per-wallet balance errors, and avoid making every browser poll trigger a full IBEX balance sweep. +- Running through `ts-node` may need `transpile-only` and `tsconfig-paths/register`, matching the earlier local script behavior. +- Large account lists should remain cheap: 60 accounts and 120 wallets is small, so simple sequential fetches are acceptable for operator clarity. + +## Dual-Model Review Notes + +- Reviewer 1 required explicit migration lookup arguments so PRE/prepared migration records remain visible. Plan updated. +- Reviewer 1 required explicit currency on balance reads. Implementation will always pass `wallet.currency`. +- Reviewer 1 required support for both manifest shapes and default count validation. Plan updated. +- Reviewer 1 required loopback-only binding. Plan updated. +- Reviewer 1 recommended keeping the dashboard out of GraphQL/production HTTP routes. The module will only be consumed by the local script and unit tests. +- Reviewer 2 required server-side snapshot caching and a slower poll interval to avoid roughly 55-60 IBEX calls/sec. Plan updated. +- Reviewer 2 required dependency injection for the snapshot builder. The builder will take manifests, repos, cutover repo, and `getBalanceForWallet`. diff --git a/docs/plans/2026-05-28-cutover-dashboard-lazy-balances.md b/docs/plans/2026-05-28-cutover-dashboard-lazy-balances.md new file mode 100644 index 000000000..4a99b5c86 --- /dev/null +++ b/docs/plans/2026-05-28-cutover-dashboard-lazy-balances.md @@ -0,0 +1,77 @@ +# Cash Wallet Cutover Dashboard Lazy Balances Implementation Plan + +> Implementation note: keep this local dashboard read-only and preserve the existing IBEX balance throttle. + +**Goal:** Make the local Cash Wallet Cutover Dashboard render readiness and account structure immediately while IBEX wallet balances hydrate lazily in the background. + +**Architecture:** Split dashboard data into a fast structural snapshot and a throttled balance refresh path. The structural snapshot reads Mongo, migrations, and preflight state, but does not call IBEX. A local in-memory balance cache and single-worker queue refresh wallet balances with the existing throttle and expose cached values through a lazy `/api/balances` endpoint. + +**Tech Stack:** TypeScript, Express, existing Flash repositories, Jest unit tests, vanilla browser JavaScript. + +--- + +## Task 1: Structural Snapshot Mode + +**Files:** +- Modify: `src/app/cash-wallet-cutover/operator-dashboard.ts` +- Test: `test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts` + +**Steps:** +1. Add a failing unit test proving `buildCashWalletCutoverOperatorSnapshot` can produce rows without calling `getBalanceForWallet` when balance mode is disabled. +2. Add a placeholder balance formatter that returns `display: "loading"`, zero minor units, and a `status` field for balance hydration. +3. Thread a `balanceMode` option through the snapshot builder with default live behavior preserved for existing callers. +4. Verify existing live-balance tests still pass. + +## Task 2: Balance Cache And Queue + +**Files:** +- Modify: `src/scripts/cash-wallet-cutover-dashboard.ts` + +**Steps:** +1. Add a focused unit-testable helper only if it can stay small; otherwise keep the cache local to the script. +2. Add an in-memory `Map` and FIFO queue with de-duping. +3. Keep the existing one-wallet-at-a-time throttle and retry behavior inside the queue worker. +4. Add cache statuses for the first pass: `loading`, `fresh`, and `error`. + +## Task 3: Lazy Balance Endpoints + +**Files:** +- Modify: `src/scripts/cash-wallet-cutover-dashboard.ts` + +**Steps:** +1. Change `/api/snapshot` to build structural snapshots only. +2. Add `GET /api/balances?walletIds=...&refresh=0|1`, returning cached balance payloads immediately and enqueueing requested wallet IDs. +3. Add `GET /api/balance-status` for queue length, refreshed count, loading count, and last sweep timestamp. +4. Keep `?refresh=1` on `/api/snapshot` as structural refresh only, not a full IBEX balance sweep. + +## Task 4: Browser Lazy Hydration + +**Files:** +- Modify: `src/scripts/cash-wallet-cutover-dashboard.ts` + +**Steps:** +1. Render the structural snapshot immediately. +2. Collect wallet IDs from the structural snapshot and hand them to `/api/balances` without blocking first render. +3. Poll `/api/balances` and update the row objects in memory as balances arrive. +4. Show status text such as `Balances 24/192 refreshed` instead of blocking `Loading...`. +5. Ensure filters continue to work while balances are loading. + +## Task 5: Verification + +**Commands:** +1. Run focused unit tests: + `PATH=/Users/dread/.nvm/versions/node/v20.20.0/bin:$PATH TEST=test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts yarn test:unit` +2. Restart local dashboard: + `tmux kill-session -t cutover-dashboard` then start the existing dashboard command. +3. Verify: + - `GET /` returns HTML. + - `GET /api/snapshot?refresh=1` returns quickly and reports `watchlistAccounts: 60`. + - `GET /api/balances` returns immediately with cached/loading payloads. + - `GET /api/balance-status` shows queue progress. + +## Constraints + +- Do not increase IBEX request rate. +- Do not mutate accounts, wallets, migrations, or cutover config. +- Keep dashboard local-only on `127.0.0.1`. +- Avoid broad refactors and generated-file churn. diff --git a/src/app/cash-wallet-cutover/index.ts b/src/app/cash-wallet-cutover/index.ts index eae3f76bd..67d632d84 100644 --- a/src/app/cash-wallet-cutover/index.ts +++ b/src/app/cash-wallet-cutover/index.ts @@ -21,3 +21,4 @@ export * from "./runtime-services" export * from "./orchestrator" export * from "./lifecycle" export * from "./preview" +export * from "./provision-usdt-wallets" diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts new file mode 100644 index 000000000..743319128 --- /dev/null +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -0,0 +1,930 @@ +import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" +import { WalletType } from "@domain/wallets" + +import { CashWalletCutoverDiscovery } from "./discovery" +import { CashWalletCutoverPreflightReport } from "./preflight" + +export type CashWalletCutoverOperatorManifestAccount = { + batchRunId?: string + index?: number + phone?: string + username?: string + accountId: AccountId + accountUuid?: AccountUuid + expectedUsdWalletId?: WalletId + expectedUsdtWalletId?: WalletId +} + +export type OperatorBalanceStatus = "loading" | "fresh" | "error" + +export type OperatorBalance = { + currency: WalletCurrency + display: string + minorUnits: string + minorUnitsNumber: number + status?: OperatorBalanceStatus + error?: string +} + +export type OperatorWallet = { + id: WalletId + currency: WalletCurrency + expected: boolean + balance: OperatorBalance +} + +export type OperatorAccount = { + batchRunId?: string + index?: number + phone?: string + username?: string + accountId: AccountId + accountUuid?: AccountUuid + expectedUsdWalletId?: WalletId + expectedUsdtWalletId?: WalletId + watchlisted: boolean + defaultWalletId?: WalletId + defaultWalletCurrency?: WalletCurrency + walletCount: number + usdWallets: OperatorWallet[] + usdtWallets: OperatorWallet[] + migrationStatus: CashWalletMigrationStatus | "none" + migrationUpdatedAt?: string + cutoverBalanceAudit?: OperatorCutoverBalanceAudit + anomalies: string[] +} + +export type OperatorCutoverBalanceAudit = { + status: "loading" | "shortfall" | "verified" + sourceUsdCents: number + expectedMinimumUsdtMicros: number + destinationStartingBalanceUsdtMicros: number + currentDestinationBalanceUsdtMicros: number + finalDeltaUsdtMicros: number + roundingSubsidyUsdtMicros: number + shortfallUsdtMicros: number +} + +export type OperatorTreasuryAccount = { + accountId: AccountId + accountUuid?: AccountUuid + role?: string + defaultWalletId?: WalletId + defaultWalletCurrency?: WalletCurrency + walletCount: number + usdWallets: OperatorWallet[] + usdtWallets: OperatorWallet[] + anomalies: string[] +} + +export type OperatorTreasurySummary = { + accounts: number + wallets: number + usdTotalCents: number + usdtTotalMicros: number +} + +export type OperatorReconciliationSummary = { + customerTotalCents: number + treasuryTotalCents: number + systemTotalCents: number +} + +export type CashWalletCutoverOperatorSnapshot = { + generatedAt: string + cutover: { + state: CashWalletCutoverState + cutoverVersion: number + runId?: string + updatedAt?: string + } + preflight?: CashWalletCutoverPreflightReport + summary: { + accounts: number + wallets: { + current: number + target: number + usd: number + usdt: number + missingUsdt: number + } + fundedUsdOnlyAccounts: number + usdTotalCents: number + usdtTotalMicros: number + anomalies: number + canStart: boolean + blockers: number + watchlistAccounts: number + migrationStatuses: Record + } + accounts: OperatorAccount[] + treasury: { + accounts: OperatorTreasuryAccount[] + summary: OperatorTreasurySummary + } + reconciliation: OperatorReconciliationSummary +} + +type AccountManifestRecord = { + index?: number + phone?: string + username?: string + accountId?: string + account?: { id?: string } + id?: string + accountUuid?: string + usdWalletId?: string + usdtWalletId?: string +} + +type ManifestShape = { + runId?: string + accounts?: AccountManifestRecord[] + created?: AccountManifestRecord[] +} + +type BuildSnapshotArgs = { + manifestAccounts: CashWalletCutoverOperatorManifestAccount[] + discoveredAccounts?: CashWalletCutoverDiscovery[] + accountsRepo: Pick + walletsRepo: Pick + migrationsRepo: { + getConfig: () => Promise + findMigrationByAccountId: (args: { + accountId: AccountId + cutoverVersion: number + runId: string + }) => Promise + } + getBalanceForWallet: (args: { + walletId: WalletId + currency?: WalletCurrency + }) => Promise + migrationLookup?: { + cutoverVersion: number + runId: string + } + preflightReport?: CashWalletCutoverPreflightReport + balanceReadAttempts?: number + balanceMode?: "live" | "structural" + treasuryAccountIds?: AccountId[] + now?: Date +} + +export const parseCashWalletCutoverOperatorManifest = ( + input: ManifestShape | AccountManifestRecord[], +): CashWalletCutoverOperatorManifestAccount[] => { + const batchRunId = Array.isArray(input) ? undefined : input.runId + const records = Array.isArray(input) ? input : (input.accounts ?? input.created ?? []) + + const accounts = records.map((record) => { + const accountId = record.accountId ?? record.account?.id ?? record.id + if (!accountId) { + throw new Error("Operator manifest record is missing accountId") + } + + return { + batchRunId, + index: record.index, + phone: record.phone, + username: record.username, + accountId: accountId as AccountId, + accountUuid: record.accountUuid as AccountUuid | undefined, + expectedUsdWalletId: record.usdWalletId as WalletId | undefined, + expectedUsdtWalletId: record.usdtWalletId as WalletId | undefined, + } + }) + + const seen = new Set() + for (const account of accounts) { + if (seen.has(account.accountId)) { + throw new Error(`Duplicate operator manifest accountId: ${account.accountId}`) + } + seen.add(account.accountId) + } + + return accounts +} + +const csvHeaders = [ + "generatedAt", + "cutoverState", + "cutoverVersion", + "cutoverRunId", + "cutoverUpdatedAt", + "watchlisted", + "batchRunId", + "index", + "phone", + "username", + "accountId", + "accountUuid", + "defaultWalletId", + "defaultWalletCurrency", + "expectedUsdWalletId", + "expectedUsdtWalletId", + "walletCount", + "usdWalletIds", + "usdBalanceDisplays", + "usdBalanceMinorUnits", + "usdBalanceStatuses", + "usdtWalletIds", + "usdtBalanceDisplays", + "usdtBalanceMinorUnits", + "usdtBalanceStatuses", + "migrationStatus", + "migrationUpdatedAt", + "cutoverBalanceAudit", + "anomalies", +] + +const csvValue = (value: unknown): string => { + if (value === undefined || value === null) return "" + const text = String(value) + return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text +} + +const walletIdsCsv = (wallets: OperatorWallet[]) => + wallets.map((wallet) => wallet.id).join(";") + +const walletBalanceDisplaysCsv = (wallets: OperatorWallet[]) => + wallets.map((wallet) => wallet.balance.display).join(";") + +const walletBalanceMinorUnitsCsv = (wallets: OperatorWallet[]) => + wallets.map((wallet) => wallet.balance.minorUnits).join(";") + +const walletBalanceStatusesCsv = (wallets: OperatorWallet[]) => + wallets.map((wallet) => wallet.balance.status ?? "").join(";") + +const cutoverBalanceAuditCsv = (audit?: OperatorCutoverBalanceAudit) => { + if (!audit) return "" + return [ + `status=${audit.status}`, + `expectedMinimumUsdtMicros=${audit.expectedMinimumUsdtMicros}`, + `finalDeltaUsdtMicros=${audit.finalDeltaUsdtMicros}`, + `roundingSubsidyUsdtMicros=${audit.roundingSubsidyUsdtMicros}`, + `shortfallUsdtMicros=${audit.shortfallUsdtMicros}`, + ].join(";") +} + +export const formatCashWalletCutoverOperatorSnapshotCsv = ( + snapshot: CashWalletCutoverOperatorSnapshot, +): string => { + const rows = snapshot.accounts.map((account) => + [ + snapshot.generatedAt, + snapshot.cutover.state, + snapshot.cutover.cutoverVersion, + snapshot.cutover.runId, + snapshot.cutover.updatedAt, + account.watchlisted, + account.batchRunId, + account.index, + account.phone, + account.username, + account.accountId, + account.accountUuid, + account.defaultWalletId, + account.defaultWalletCurrency, + account.expectedUsdWalletId, + account.expectedUsdtWalletId, + account.walletCount, + walletIdsCsv(account.usdWallets), + walletBalanceDisplaysCsv(account.usdWallets), + walletBalanceMinorUnitsCsv(account.usdWallets), + walletBalanceStatusesCsv(account.usdWallets), + walletIdsCsv(account.usdtWallets), + walletBalanceDisplaysCsv(account.usdtWallets), + walletBalanceMinorUnitsCsv(account.usdtWallets), + walletBalanceStatusesCsv(account.usdtWallets), + account.migrationStatus, + account.migrationUpdatedAt, + cutoverBalanceAuditCsv(account.cutoverBalanceAudit), + account.anomalies.join(";"), + ].map(csvValue), + ) + + return [csvHeaders, ...rows].map((row) => row.join(",")).join("\n") +} + +const describeError = (error: Error): string => { + const message = error.message?.split("\n")[0]?.trim() + if (message) return message + + if (error.name && error.name !== "Error") return error.name + + const rendered = String(error) + return rendered && rendered !== "[object Object]" ? rendered : "Unknown error" +} + +const balanceError = (wallet: Wallet, error: Error): OperatorBalance => ({ + currency: wallet.currency, + display: "error", + minorUnits: "0", + minorUnitsNumber: 0, + status: "error", + error: describeError(error), +}) + +export const formatOperatorBalance = ( + wallet: Wallet, + balance: USDAmount | USDTAmount | ApplicationError, +): OperatorBalance => { + if (balance instanceof Error) return balanceError(wallet, balance) + + if (wallet.currency === WalletCurrency.Usdt && balance instanceof USDTAmount) { + const micros = balance.asSmallestUnits() + return { + currency: WalletCurrency.Usdt, + display: `${(Number(micros) / 1_000_000).toFixed(2)} USDT`, + minorUnits: micros, + minorUnitsNumber: Number(micros), + status: "fresh", + } + } + + if (wallet.currency === WalletCurrency.Usd && balance instanceof USDAmount) { + const cents = balance.asCents() + return { + currency: WalletCurrency.Usd, + display: `$${balance.asDollars(2)}`, + minorUnits: cents, + minorUnitsNumber: Number(cents), + status: "fresh", + } + } + + return { + currency: wallet.currency, + display: "unexpected currency", + minorUnits: "0", + minorUnitsNumber: 0, + status: "error", + error: `Expected ${wallet.currency} balance`, + } +} + +const loadingBalance = (wallet: Wallet): OperatorBalance => ({ + currency: wallet.currency, + display: "loading", + minorUnits: "0", + minorUnitsNumber: 0, + status: "loading", +}) + +const summarizeWallet = async ({ + wallet, + expectedWalletId, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, +}: { + wallet: Wallet + expectedWalletId?: WalletId + getBalanceForWallet: BuildSnapshotArgs["getBalanceForWallet"] + balanceReadAttempts: number + balanceMode: BuildSnapshotArgs["balanceMode"] +}): Promise => { + if (balanceMode === "structural") { + return { + id: wallet.id, + currency: wallet.currency, + expected: expectedWalletId === undefined || expectedWalletId === wallet.id, + balance: loadingBalance(wallet), + } + } + + let balance: USDAmount | USDTAmount | ApplicationError = new Error( + "Balance read was not attempted", + ) as ApplicationError + + for (let attempt = 0; attempt < balanceReadAttempts; attempt++) { + balance = await getBalanceForWallet({ + walletId: wallet.id, + currency: wallet.currency, + }) + if (!(balance instanceof Error)) break + } + + return { + id: wallet.id, + currency: wallet.currency, + expected: expectedWalletId === undefined || expectedWalletId === wallet.id, + balance: formatOperatorBalance(wallet, balance), + } +} + +const increment = (record: Record, key: string) => { + record[key] = (record[key] ?? 0) + 1 +} + +const accountsForDashboard = ({ + manifestAccounts, + discoveredAccounts, +}: { + manifestAccounts: CashWalletCutoverOperatorManifestAccount[] + discoveredAccounts?: CashWalletCutoverDiscovery[] +}): OperatorAccountInput[] => { + const manifestByAccountId = new Map( + manifestAccounts.map((account) => [account.accountId, account]), + ) + + if (!discoveredAccounts) { + return manifestAccounts.map((account) => ({ ...account, watchlisted: true })) + } + + const merged = discoveredAccounts.map((discovery) => { + const manifestAccount = manifestByAccountId.get(discovery.accountId) + return { + ...manifestAccount, + accountId: discovery.accountId, + accountUuid: manifestAccount?.accountUuid ?? discovery.accountUuid, + expectedUsdWalletId: + manifestAccount?.expectedUsdWalletId ?? discovery.legacyUsdWalletId, + expectedUsdtWalletId: + manifestAccount?.expectedUsdtWalletId ?? discovery.destinationUsdtWalletId, + watchlisted: manifestAccount !== undefined, + } + }) + + const discoveredAccountIds = new Set(merged.map((account) => account.accountId)) + const missingManifestAccounts = manifestAccounts + .filter((account) => !discoveredAccountIds.has(account.accountId)) + .map((account) => ({ ...account, watchlisted: true })) + + return [...merged, ...missingManifestAccounts] +} + +type OperatorAccountInput = CashWalletCutoverOperatorManifestAccount & { + watchlisted: boolean +} + +const usdTotalCentsForAccounts = ( + accounts: Array<{ usdWallets: OperatorWallet[] }>, +): number => + accounts.reduce( + (sum, account) => + sum + + account.usdWallets.reduce( + (walletSum, wallet) => walletSum + wallet.balance.minorUnitsNumber, + 0, + ), + 0, + ) + +const usdtTotalMicrosForAccounts = ( + accounts: Array<{ usdtWallets: OperatorWallet[] }>, +): number => + accounts.reduce( + (sum, account) => + sum + + account.usdtWallets.reduce( + (walletSum, wallet) => walletSum + wallet.balance.minorUnitsNumber, + 0, + ), + 0, + ) + +const parseIntegerAmount = (value?: string): number | undefined => { + if (value === undefined || !/^\d+$/.test(value)) return undefined + return Number(value) +} + +const computeCutoverBalanceAudit = ({ + migration, + usdtWallets, +}: { + migration?: CashWalletMigration | null + usdtWallets: OperatorWallet[] +}): OperatorCutoverBalanceAudit | undefined => { + if (!migration || migration.status !== "complete") return undefined + + const sourceUsdCents = parseIntegerAmount(migration.sourceBalanceUsdCents) + const expectedMinimumUsdtMicros = parseIntegerAmount( + migration.destinationAmountUsdtMicros, + ) + const destinationStartingBalanceUsdtMicros = parseIntegerAmount( + migration.destinationStartingBalanceUsdtMicros, + ) + + if ( + sourceUsdCents === undefined || + expectedMinimumUsdtMicros === undefined || + destinationStartingBalanceUsdtMicros === undefined + ) { + return undefined + } + + const destinationWallet = usdtWallets.find( + (wallet) => wallet.id === migration.destinationUsdtWalletId, + ) + if (!destinationWallet) return undefined + + const currentDestinationBalanceUsdtMicros = + destinationWallet.balance.minorUnitsNumber + const finalDeltaUsdtMicros = Math.max( + 0, + currentDestinationBalanceUsdtMicros - destinationStartingBalanceUsdtMicros, + ) + const shortfallUsdtMicros = Math.max( + 0, + expectedMinimumUsdtMicros - finalDeltaUsdtMicros, + ) + const roundingSubsidyUsdtMicros = Math.max( + 0, + finalDeltaUsdtMicros - expectedMinimumUsdtMicros, + ) + + return { + status: + destinationWallet.balance.status === "loading" + ? "loading" + : shortfallUsdtMicros > 0 + ? "shortfall" + : "verified", + sourceUsdCents, + expectedMinimumUsdtMicros, + destinationStartingBalanceUsdtMicros, + currentDestinationBalanceUsdtMicros, + finalDeltaUsdtMicros, + roundingSubsidyUsdtMicros, + shortfallUsdtMicros, + } +} + +export const refreshOperatorAccountCutoverBalanceAudit = < + T extends { + expectedUsdtWalletId?: WalletId + usdtWallets: OperatorWallet[] + cutoverBalanceAudit?: OperatorCutoverBalanceAudit + }, +>( + account: T, +): T => { + const audit = account.cutoverBalanceAudit + if (!audit) return account + + const destinationWallet = + account.usdtWallets.find((wallet) => wallet.id === account.expectedUsdtWalletId) ?? + account.usdtWallets.find((wallet) => wallet.expected) ?? + account.usdtWallets[0] + if (!destinationWallet) return account + + const currentDestinationBalanceUsdtMicros = + destinationWallet.balance.minorUnitsNumber + const finalDeltaUsdtMicros = Math.max( + 0, + currentDestinationBalanceUsdtMicros - + audit.destinationStartingBalanceUsdtMicros, + ) + const shortfallUsdtMicros = Math.max( + 0, + audit.expectedMinimumUsdtMicros - finalDeltaUsdtMicros, + ) + const roundingSubsidyUsdtMicros = Math.max( + 0, + finalDeltaUsdtMicros - audit.expectedMinimumUsdtMicros, + ) + + return { + ...account, + cutoverBalanceAudit: { + ...audit, + status: + destinationWallet.balance.status === "loading" + ? "loading" + : shortfallUsdtMicros > 0 + ? "shortfall" + : "verified", + currentDestinationBalanceUsdtMicros, + finalDeltaUsdtMicros, + roundingSubsidyUsdtMicros, + shortfallUsdtMicros, + }, + } +} + +const combinedTotalCents = ({ + usdTotalCents, + usdtTotalMicros, +}: { + usdTotalCents: number + usdtTotalMicros: number +}) => usdTotalCents + usdtTotalMicros / 10_000 + +const treasurySummary = ( + accounts: OperatorTreasuryAccount[], +): OperatorTreasurySummary => ({ + accounts: accounts.length, + wallets: accounts.reduce((sum, account) => sum + account.walletCount, 0), + usdTotalCents: usdTotalCentsForAccounts(accounts), + usdtTotalMicros: usdtTotalMicrosForAccounts(accounts), +}) + +const reconciliationSummary = ({ + customerUsdTotalCents, + customerUsdtTotalMicros, + treasury, +}: { + customerUsdTotalCents: number + customerUsdtTotalMicros: number + treasury: OperatorTreasurySummary +}): OperatorReconciliationSummary => { + const customerTotalCents = combinedTotalCents({ + usdTotalCents: customerUsdTotalCents, + usdtTotalMicros: customerUsdtTotalMicros, + }) + const treasuryTotalCents = combinedTotalCents({ + usdTotalCents: treasury.usdTotalCents, + usdtTotalMicros: treasury.usdtTotalMicros, + }) + return { + customerTotalCents, + treasuryTotalCents, + systemTotalCents: customerTotalCents + treasuryTotalCents, + } +} + +const summarizeTreasuryAccount = async ({ + accountId, + accountsRepo, + walletsRepo, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, +}: { + accountId: AccountId + accountsRepo: BuildSnapshotArgs["accountsRepo"] + walletsRepo: BuildSnapshotArgs["walletsRepo"] + getBalanceForWallet: BuildSnapshotArgs["getBalanceForWallet"] + balanceReadAttempts: number + balanceMode: BuildSnapshotArgs["balanceMode"] +}): Promise => { + const account = await accountsRepo.findById(accountId) + if (account instanceof Error) { + return { + accountId, + walletCount: 0, + usdWallets: [], + usdtWallets: [], + anomalies: ["missing_account"], + } + } + + const rawWallets = await walletsRepo.listByAccountId(account.id) + if (rawWallets instanceof Error) throw rawWallets + + const cashWallets = rawWallets.filter((wallet) => wallet.type === WalletType.Checking) + const usdWalletsRaw = cashWallets.filter( + (wallet) => wallet.currency === WalletCurrency.Usd, + ) + const usdtWalletsRaw = cashWallets.filter( + (wallet) => wallet.currency === WalletCurrency.Usdt, + ) + const defaultWallet = cashWallets.find((wallet) => wallet.id === account.defaultWalletId) + + const [usdWallets, usdtWallets] = await Promise.all([ + Promise.all( + usdWalletsRaw.map((wallet) => + summarizeWallet({ + wallet, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, + }), + ), + ), + Promise.all( + usdtWalletsRaw.map((wallet) => + summarizeWallet({ + wallet, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, + }), + ), + ), + ]) + + return { + accountId: account.id, + accountUuid: account.uuid, + role: account.role ?? "funder", + defaultWalletId: account.defaultWalletId, + defaultWalletCurrency: defaultWallet?.currency, + walletCount: rawWallets.length, + usdWallets, + usdtWallets, + anomalies: [...usdWallets, ...usdtWallets].some( + (wallet) => wallet.balance.error !== undefined, + ) + ? ["balance_error"] + : [], + } +} + +export const buildCashWalletCutoverOperatorSnapshot = async ({ + manifestAccounts, + discoveredAccounts, + accountsRepo, + walletsRepo, + migrationsRepo, + getBalanceForWallet, + migrationLookup, + preflightReport, + balanceReadAttempts = 1, + balanceMode = "live", + treasuryAccountIds = [], + now = new Date(), +}: BuildSnapshotArgs): Promise => { + const config = await migrationsRepo.getConfig() + if (config instanceof Error) throw config + const lookup = + migrationLookup ?? + (config.runId + ? { + cutoverVersion: config.cutoverVersion, + runId: config.runId, + } + : undefined) + + const accounts: OperatorAccount[] = [] + const migrationStatuses: Record = {} + const operatorAccounts = accountsForDashboard({ manifestAccounts, discoveredAccounts }) + + for (const dashboardAccount of operatorAccounts) { + const anomalies: string[] = [] + const account = await accountsRepo.findById(dashboardAccount.accountId) + if (account instanceof Error) { + increment(migrationStatuses, "none") + accounts.push({ + ...dashboardAccount, + walletCount: 0, + usdWallets: [], + usdtWallets: [], + migrationStatus: "none", + anomalies: ["missing_account"], + }) + continue + } + + const rawWallets = await walletsRepo.listByAccountId(account.id) + if (rawWallets instanceof Error) throw rawWallets + + const cashWallets = rawWallets.filter((wallet) => wallet.type === WalletType.Checking) + const usdWalletsRaw = cashWallets.filter( + (wallet) => wallet.currency === WalletCurrency.Usd, + ) + const usdtWalletsRaw = cashWallets.filter( + (wallet) => wallet.currency === WalletCurrency.Usdt, + ) + const defaultWallet = cashWallets.find( + (wallet) => wallet.id === account.defaultWalletId, + ) + + if (usdWalletsRaw.length === 0) anomalies.push("missing_usd") + if (usdtWalletsRaw.length === 0) anomalies.push("missing_usdt") + if (usdWalletsRaw.length > 1) anomalies.push("duplicate_usd") + if (usdtWalletsRaw.length > 1) anomalies.push("duplicate_usdt") + if (!defaultWallet) anomalies.push("default_not_cash") + + const [usdWallets, usdtWallets] = await Promise.all([ + Promise.all( + usdWalletsRaw.map((wallet) => + summarizeWallet({ + wallet, + expectedWalletId: dashboardAccount.expectedUsdWalletId, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, + }), + ), + ), + Promise.all( + usdtWalletsRaw.map((wallet) => + summarizeWallet({ + wallet, + expectedWalletId: dashboardAccount.expectedUsdtWalletId, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, + }), + ), + ), + ]) + + if ( + [...usdWallets, ...usdtWallets].some((wallet) => wallet.balance.error !== undefined) + ) { + anomalies.push("balance_error") + } + if ([...usdWallets, ...usdtWallets].some((wallet) => !wallet.expected)) { + anomalies.push("unexpected_wallet_id") + } + + let migrationStatus: CashWalletMigrationStatus | "none" = "none" + let migrationUpdatedAt: string | undefined + let migration: CashWalletMigration | null = null + if (lookup) { + const migrationResult = await migrationsRepo.findMigrationByAccountId({ + accountId: account.id, + cutoverVersion: lookup.cutoverVersion, + runId: lookup.runId, + }) + if (migrationResult instanceof Error) throw migrationResult + migration = migrationResult + if (migration) { + migrationStatus = migration.status + migrationUpdatedAt = migration.updatedAt?.toISOString() + if (migration.status === "failed") anomalies.push("migration_failed") + if (migration.status === "requires_operator_review") { + anomalies.push("migration_requires_review") + } + } + } + increment(migrationStatuses, migrationStatus) + + accounts.push({ + ...dashboardAccount, + accountUuid: account.uuid ?? dashboardAccount.accountUuid, + defaultWalletId: account.defaultWalletId, + defaultWalletCurrency: defaultWallet?.currency, + walletCount: rawWallets.length, + usdWallets, + usdtWallets, + migrationStatus, + migrationUpdatedAt, + cutoverBalanceAudit: computeCutoverBalanceAudit({ migration, usdtWallets }), + anomalies, + }) + } + + const treasuryAccounts = await Promise.all( + treasuryAccountIds.map((accountId) => + summarizeTreasuryAccount({ + accountId, + accountsRepo, + walletsRepo, + getBalanceForWallet, + balanceReadAttempts, + balanceMode, + }), + ), + ) + const treasury = treasurySummary(treasuryAccounts) + const usdTotalCents = usdTotalCentsForAccounts(accounts) + const usdtTotalMicros = usdtTotalMicrosForAccounts(accounts) + const missingUsdt = accounts.filter( + (account) => account.usdtWallets.length === 0, + ).length + const blockers = accounts.filter( + (account) => + account.anomalies.includes("missing_usd") || + account.anomalies.includes("missing_usdt"), + ).length + + const reconciliation = reconciliationSummary({ + customerUsdTotalCents: usdTotalCents, + customerUsdtTotalMicros: usdtTotalMicros, + treasury, + }) + + return { + generatedAt: now.toISOString(), + cutover: { + state: config.state, + cutoverVersion: config.cutoverVersion, + runId: config.runId, + updatedAt: config.updatedAt?.toISOString(), + }, + preflight: preflightReport, + summary: { + accounts: accounts.length, + wallets: { + current: accounts.reduce((sum, account) => sum + account.walletCount, 0), + target: accounts.length * 2, + usd: accounts.reduce((sum, account) => sum + account.usdWallets.length, 0), + usdt: accounts.reduce((sum, account) => sum + account.usdtWallets.length, 0), + missingUsdt, + }, + fundedUsdOnlyAccounts: accounts.filter( + (account) => + account.usdtWallets.length === 0 && + account.usdWallets.some((wallet) => wallet.balance.minorUnitsNumber > 0), + ).length, + usdTotalCents, + usdtTotalMicros, + anomalies: accounts.filter((account) => account.anomalies.length > 0).length, + canStart: blockers === 0, + blockers, + watchlistAccounts: accounts.filter((account) => account.watchlisted).length, + migrationStatuses, + }, + accounts, + treasury: { + accounts: treasuryAccounts, + summary: treasury, + }, + reconciliation, + } +} diff --git a/src/app/cash-wallet-cutover/orchestrator.ts b/src/app/cash-wallet-cutover/orchestrator.ts index 89d8203c6..3e58b17c6 100644 --- a/src/app/cash-wallet-cutover/orchestrator.ts +++ b/src/app/cash-wallet-cutover/orchestrator.ts @@ -19,6 +19,7 @@ export const runPrimaryCashWalletCutoverBatch = ({ runId, workerId, limit, + stepDelayMs, lockStaleBefore, migrationsRepo = CashWalletCutoverRepository(), runtimeServices = createCashWalletMigrationRuntimeServices(), @@ -27,6 +28,7 @@ export const runPrimaryCashWalletCutoverBatch = ({ runId: string workerId: string limit?: number + stepDelayMs?: number lockStaleBefore: Date migrationsRepo?: PrimaryCashWalletCutoverBatchRepository runtimeServices?: PrimaryCashWalletCutoverRuntimeServices @@ -41,6 +43,7 @@ export const runPrimaryCashWalletCutoverBatch = ({ runId, workerId, limit, + stepDelayMs, lockStaleBefore, migrationsRepo, executor: (migration) => diff --git a/src/app/cash-wallet-cutover/provision-usdt-wallets.ts b/src/app/cash-wallet-cutover/provision-usdt-wallets.ts new file mode 100644 index 000000000..94c4979f7 --- /dev/null +++ b/src/app/cash-wallet-cutover/provision-usdt-wallets.ts @@ -0,0 +1,143 @@ +import { WalletCurrency } from "@domain/shared" +import { WalletType } from "@domain/wallets" + +import { discoverCashWalletCutoverAccounts } from "./discovery" +import { InvalidCashWalletCutoverStateTransitionError } from "./errors" +import { + buildCashWalletCutoverPreflightReport, + CashWalletCutoverPreflightReport, +} from "./preflight" + +type ProvisionedCashWalletUsdtWallet = { + accountId: AccountId + walletId?: WalletId +} + +type FailedCashWalletUsdtWalletProvision = { + accountId: AccountId + error: string +} + +type ProvisionPrimaryCashWalletUsdtWalletsResult = { + before: CashWalletCutoverPreflightReport + after: CashWalletCutoverPreflightReport + eligible: number + provisioned: ProvisionedCashWalletUsdtWallet[] + failed: FailedCashWalletUsdtWalletProvision[] + dryRun: boolean +} + +type CashWalletCutoverProvisioningRepository = { + getConfig: () => Promise +} + +type AddWalletIfNonexistent = ({ + accountId, + type, + currency, +}: { + accountId: AccountId + type: WalletType + currency: WalletCurrency +}) => Promise + +const defaultSleep = (delayMs: number) => + new Promise((resolve) => setTimeout(resolve, delayMs)) + +const errorMessage = (error: unknown): string => { + if (error instanceof Error && error.message) return error.message + return String(error) +} + +export type { ProvisionPrimaryCashWalletUsdtWalletsResult } + +export const provisionPrimaryCashWalletUsdtWallets = async ({ + cutoverVersion, + runId, + accountsRepo, + walletsRepo, + migrationsRepo, + addWalletIfNonexistent, + provisionLimit, + provisionDelayMs = 0, + dryRun = false, + sleep = defaultSleep, +}: { + cutoverVersion: number + runId: string + accountsRepo: Pick + walletsRepo: Pick + migrationsRepo: CashWalletCutoverProvisioningRepository + addWalletIfNonexistent: AddWalletIfNonexistent + provisionLimit?: number + provisionDelayMs?: number + dryRun?: boolean + sleep?: (delayMs: number) => Promise +}): Promise => { + const config = await migrationsRepo.getConfig() + if (config instanceof Error) return config + + if (config.state !== "pre") { + return new InvalidCashWalletCutoverStateTransitionError( + "Cash wallet USDT provisioning can only run before cutover start", + ) + } + + const discoveries = await discoverCashWalletCutoverAccounts({ + accountsRepo, + walletsRepo, + }) + if (discoveries instanceof Error) return discoveries + + const before = buildCashWalletCutoverPreflightReport({ + cutoverVersion, + runId, + discoveries, + }) + + const eligibleDiscoveries = discoveries + .filter(({ status }) => status === "missing_destination_usdt") + .slice(0, provisionLimit) + const provisioned: ProvisionedCashWalletUsdtWallet[] = [] + const failed: FailedCashWalletUsdtWalletProvision[] = [] + + if (!dryRun) { + for (const [index, discovery] of eligibleDiscoveries.entries()) { + const wallet = await addWalletIfNonexistent({ + accountId: discovery.accountId, + type: WalletType.Checking, + currency: WalletCurrency.Usdt, + }) + + if (wallet instanceof Error) { + failed.push({ accountId: discovery.accountId, error: errorMessage(wallet) }) + } else { + provisioned.push({ accountId: discovery.accountId, walletId: wallet.id }) + } + + if (provisionDelayMs > 0 && index < eligibleDiscoveries.length - 1) { + await sleep(provisionDelayMs) + } + } + } + + const afterDiscoveries = dryRun + ? discoveries + : await discoverCashWalletCutoverAccounts({ accountsRepo, walletsRepo }) + if (afterDiscoveries instanceof Error) return afterDiscoveries + + const after = buildCashWalletCutoverPreflightReport({ + cutoverVersion, + runId, + discoveries: afterDiscoveries, + }) + + return { + before, + after, + eligible: eligibleDiscoveries.length, + provisioned, + failed, + dryRun, + } +} diff --git a/src/app/cash-wallet-cutover/runner.ts b/src/app/cash-wallet-cutover/runner.ts index c937496f0..70789c8c9 100644 --- a/src/app/cash-wallet-cutover/runner.ts +++ b/src/app/cash-wallet-cutover/runner.ts @@ -38,6 +38,11 @@ type CashWalletMigrationBatchResult = { skipped: number } +type SleepFn = (delayMs: number) => Promise + +const sleep = (delayMs: number) => + new Promise((resolve) => setTimeout(resolve, delayMs)) + const AMBIGUOUS_SIDE_EFFECT_STATUSES: CashWalletMigrationStatus[] = [ "invoice_created", "balance_move_sending", @@ -62,6 +67,8 @@ export const runCashWalletMigrationBatch = async ({ lockStaleBefore, migrationsRepo, executor, + stepDelayMs = 0, + sleep: sleepFn = sleep, }: { cutoverVersion: number runId: string @@ -70,6 +77,8 @@ export const runCashWalletMigrationBatch = async ({ lockStaleBefore: Date migrationsRepo: CashWalletMigrationBatchRepository executor: CashWalletMigrationBatchExecutor + stepDelayMs?: number + sleep?: SleepFn }): Promise => { const migrations = await migrationsRepo.listRunnableMigrations({ cutoverVersion, @@ -85,7 +94,7 @@ export const runCashWalletMigrationBatch = async ({ skipped: 0, } - for (const migration of migrations) { + for (const [index, migration] of migrations.entries()) { result.attempted += 1 const locked = await migrationsRepo.acquireMigrationLock({ @@ -123,6 +132,10 @@ export const runCashWalletMigrationBatch = async ({ cutoverVersion, runId, }) + + if (stepDelayMs > 0 && index < migrations.length - 1) { + await sleepFn(stepDelayMs) + } } return result diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index 776149ba5..e854e1cf9 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -1,8 +1,5 @@ import { addWalletIfNonexistent, updateDefaultWalletId } from "@app/accounts" -import { - addInvoiceForRecipientForUsdWallet, - getBalanceForWallet, -} from "@app/wallets" +import { getBalanceForWallet } from "@app/wallets" import { decodeInvoice } from "@domain/bitcoin/lightning" import { InvalidWalletId } from "@domain/errors" import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" @@ -26,7 +23,7 @@ type RuntimeServiceDependencies = { addWalletIfNonexistent?: typeof addWalletIfNonexistent updateDefaultWalletId?: typeof updateDefaultWalletId getBalanceForWallet?: typeof getBalanceForWallet - createInvoice?: typeof addInvoiceForRecipientForUsdWallet + createInvoice?: typeof Ibex.addInvoice createNoAmountInvoice?: typeof Ibex.addInvoice payInvoice?: typeof Ibex.payInvoice accountsRepo?: Pick, "findById"> @@ -55,7 +52,7 @@ export const createCashWalletMigrationRuntimeServices = ( const addWallet = deps.addWalletIfNonexistent ?? addWalletIfNonexistent const updateDefaultWallet = deps.updateDefaultWalletId ?? updateDefaultWalletId const balanceForWallet = deps.getBalanceForWallet ?? getBalanceForWallet - const invoiceForRecipient = deps.createInvoice ?? addInvoiceForRecipientForUsdWallet + const invoiceForRecipient = deps.createInvoice ?? Ibex.addInvoice const noAmountInvoiceForRecipient = deps.createNoAmountInvoice ?? Ibex.addInvoice const payInvoice = deps.payInvoice ?? Ibex.payInvoice const accountsRepo = deps.accountsRepo ?? AccountsRepository() @@ -117,12 +114,17 @@ export const createCashWalletMigrationRuntimeServices = ( recipientWalletId: WalletId amount: string memo: string - }) => - invoiceForRecipient({ - recipientWalletId, - amount: amount as FractionalCentAmount, + }) => { + const usdtAmount = USDTAmount.smallestUnits(amount) + if (usdtAmount instanceof Error) return Promise.resolve(usdtAmount) + + return invoiceForRecipient({ + accountId: recipientWalletId as IbexAccountId, + amount: usdtAmount, memo, - }), + expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, + }).then(ibexInvoiceToDomainInvoice) + }, createNoAmountInvoice: ({ recipientWalletId, memo, diff --git a/src/app/cash-wallet-cutover/worker.ts b/src/app/cash-wallet-cutover/worker.ts index 67b26250f..50379e31e 100644 --- a/src/app/cash-wallet-cutover/worker.ts +++ b/src/app/cash-wallet-cutover/worker.ts @@ -1,5 +1,9 @@ import { assertCanTransition } from "./state-machine" -import { usdCentsToUsdtMicros, usdtMicrosToUsdCentsCeil } from "./amount-conversion" +import { + feeUsdCentsToUsdtMicros, + usdCentsToUsdtMicros, + usdtMicrosToUsdCentsCeil, +} from "./amount-conversion" import { InvalidCashWalletCutoverAmountError, InvalidCashWalletMigrationTransitionError, @@ -304,6 +308,10 @@ export const createCashWalletMigrationFeeReimbursementInvoice = async ({ const feeAmountUsdCents = usdtMicrosToUsdCentsCeil(feeAmountUsdtMicros) if (feeAmountUsdCents instanceof Error) return feeAmountUsdCents + const reimbursableFeeAmountUsdtMicros = feeUsdCentsToUsdtMicros(feeAmountUsdCents) + if (reimbursableFeeAmountUsdtMicros instanceof Error) + return reimbursableFeeAmountUsdtMicros + const transition = assertCanTransition( migration.status, "fee_reimbursement_invoice_created", @@ -312,7 +320,7 @@ export const createCashWalletMigrationFeeReimbursementInvoice = async ({ const invoice = await invoiceService.createInvoice({ recipientWalletId: migration.destinationUsdtWalletId, - amount: feeAmountUsdtMicros, + amount: reimbursableFeeAmountUsdtMicros, memo: `cash-wallet-cutover:${migration.runId}:${migration.id}:fee-reimbursement`, }) if (invoice instanceof Error) return invoice diff --git a/src/scripts/cash-wallet-cutover-dashboard.ts b/src/scripts/cash-wallet-cutover-dashboard.ts new file mode 100644 index 000000000..3db149e0b --- /dev/null +++ b/src/scripts/cash-wallet-cutover-dashboard.ts @@ -0,0 +1,973 @@ +#!/usr/bin/env node + +import fs from "fs" +import http from "http" + +import express from "express" +import yargs from "yargs" +import { hideBin } from "yargs/helpers" + +import { + buildCashWalletCutoverOperatorSnapshot, + CashWalletCutoverOperatorManifestAccount, + CashWalletCutoverOperatorSnapshot, + formatCashWalletCutoverOperatorSnapshotCsv, + formatOperatorBalance, + OperatorBalance, + parseCashWalletCutoverOperatorManifest, + refreshOperatorAccountCutoverBalanceAudit, +} from "@app/cash-wallet-cutover/operator-dashboard" +import { discoverCashWalletCutoverAccounts } from "@app/cash-wallet-cutover/discovery" +import { buildCashWalletCutoverPreflightReport } from "@app/cash-wallet-cutover/preflight" +import { getBalanceForWallet } from "@app/wallets" +import { WalletCurrency } from "@domain/shared" +import { setupMongoConnection } from "@services/mongodb" +import { + AccountsRepository, + CashWalletCutoverRepository, + WalletsRepository, +} from "@services/mongoose" +import { baseLogger } from "@services/logger" +import { getFunderWalletId } from "@services/ledger/caching" + +const DEFAULT_MANIFESTS = [ + "/tmp/eng345usd-20260526115410-local-backend-accounts.json", + "/tmp/eng345usdonly-20260526195758-accounts.json", +] + +const BALANCE_TIMEOUT_MS = 7_500 +const BALANCE_READ_ATTEMPTS = 3 +const BALANCE_READ_SPACING_MS = 1_000 + +const args = yargs(hideBin(process.argv)) + .option("port", { type: "number", default: 3450 }) + .option("manifest", { type: "array", string: true, default: DEFAULT_MANIFESTS }) + .option("expected-accounts", { type: "number", default: 60 }) + .option("snapshot-ttl-ms", { type: "number", default: 5_000 }) + .option("run-id", { type: "string" }) + .option("cutover-version", { type: "number" }) + .option("configPath", { type: "string", demandOption: true }) + .parseSync() + +const readManifestAccounts = (): CashWalletCutoverOperatorManifestAccount[] => { + const accounts = args.manifest.flatMap((manifestPath) => + parseCashWalletCutoverOperatorManifest( + JSON.parse(fs.readFileSync(manifestPath, "utf8")), + ), + ) + + const seen = new Set() + for (const account of accounts) { + if (seen.has(account.accountId)) { + throw new Error(`Duplicate operator dashboard accountId: ${account.accountId}`) + } + seen.add(account.accountId) + } + + if (args["expected-accounts"] && accounts.length !== args["expected-accounts"]) { + throw new Error( + `Expected ${args["expected-accounts"]} operator accounts, loaded ${accounts.length}`, + ) + } + + return accounts +} + +const withBalanceTimeout = (balance: ReturnType) => + Promise.race([ + balance, + new Promise((resolve) => { + setTimeout( + () => resolve(new Error("Balance read timed out") as ApplicationError), + BALANCE_TIMEOUT_MS, + ) + }), + ]) + +let nextBalanceReadAt = 0 + +const readBalanceThrottled = async ( + request: Parameters[0], +) => { + const now = Date.now() + const scheduledAt = Math.max(now, nextBalanceReadAt) + nextBalanceReadAt = scheduledAt + BALANCE_READ_SPACING_MS + + const waitMs = scheduledAt - now + if (waitMs > 0) { + await new Promise((resolve) => setTimeout(resolve, waitMs)) + } + + return withBalanceTimeout(getBalanceForWallet(request)) +} + +const shortId = (value?: string) => (value ? value.slice(0, 8) : "-") + +type CachedBalance = OperatorBalance & { + walletId: WalletId + updatedAt?: string +} + +const html = ` + + + + + Cash Wallet Cutover Dashboard + + + +
+
+

Cash Wallet Cutover Dashboard

+
Raw Mongo wallets plus lazy IBEX balances. Presentation filtering is bypassed.
+
+
+ canStart: - + Loading... + + +
+
+
+
+
+ + + + + + +
+
+ + + + + + + + + + + + + + + + + +
#PhoneAccountDefaultUSDUSD BalanceUSDTUSDT BalanceAuditMigrationAnomalies
+
+
+ + +` + +const start = async () => { + const manifestAccounts = readManifestAccounts() + const accountsRepo = AccountsRepository() + const walletsRepo = WalletsRepository() + const migrationsRepo = CashWalletCutoverRepository() + const migrationLookup = + args["run-id"] && args["cutover-version"] + ? { runId: args["run-id"], cutoverVersion: args["cutover-version"] } + : undefined + + await setupMongoConnection() + + const loadTreasuryAccountIds = async (): Promise => { + const funderWalletId = await getFunderWalletId() + + const funderWallet = await walletsRepo.findById(funderWalletId) + if (funderWallet instanceof Error) throw funderWallet + + return [funderWallet.accountId] + } + + const treasuryAccountIds = await loadTreasuryAccountIds() + + let cache: + | { + snapshot: CashWalletCutoverOperatorSnapshot + cachedAt: number + } + | undefined + let pending: Promise | undefined + const walletCurrencies = new Map() + const balanceCache = new Map() + const balanceQueue: Array<{ walletId: WalletId; currency: WalletCurrency }> = [] + const queuedBalanceIds = new Set() + let activeBalanceId: WalletId | undefined + let balanceWorker: Promise | undefined + + const registerSnapshotWallets = (snapshot: CashWalletCutoverOperatorSnapshot) => { + for (const account of [...snapshot.accounts, ...snapshot.treasury.accounts]) { + for (const wallet of [...account.usdWallets, ...account.usdtWallets]) { + walletCurrencies.set(wallet.id, wallet.currency) + if (!balanceCache.has(wallet.id)) { + balanceCache.set(wallet.id, { + walletId: wallet.id, + currency: wallet.currency, + display: "loading", + minorUnits: "0", + minorUnitsNumber: 0, + status: "loading", + }) + } + } + } + } + + const runBalanceWorker = () => { + if (balanceWorker) return balanceWorker + + balanceWorker = (async () => { + while (balanceQueue.length > 0) { + const request = balanceQueue.shift() + if (!request) continue + + queuedBalanceIds.delete(request.walletId) + activeBalanceId = request.walletId + balanceCache.set(request.walletId, { + walletId: request.walletId, + currency: request.currency, + display: "loading", + minorUnits: "0", + minorUnitsNumber: 0, + status: "loading", + }) + + const balance = await readBalanceThrottled({ + walletId: request.walletId, + currency: request.currency, + }) + balanceCache.set(request.walletId, { + walletId: request.walletId, + ...formatOperatorBalance( + { id: request.walletId, currency: request.currency } as Wallet, + balance, + ), + updatedAt: new Date().toISOString(), + }) + } + })().finally(() => { + activeBalanceId = undefined + balanceWorker = undefined + if (balanceQueue.length > 0) runBalanceWorker() + }) + + return balanceWorker + } + + const enqueueBalance = ({ + walletId, + currency, + force, + }: { + walletId: WalletId + currency: WalletCurrency + force: boolean + }) => { + const cached = balanceCache.get(walletId) + if (!force && cached && cached.status !== "loading") return + if (queuedBalanceIds.has(walletId) || activeBalanceId === walletId) return + + queuedBalanceIds.add(walletId) + balanceQueue.push({ walletId, currency }) + runBalanceWorker() + } + + const snapshotWithCachedBalances = ( + currentSnapshot: CashWalletCutoverOperatorSnapshot, + ): CashWalletCutoverOperatorSnapshot => ({ + ...currentSnapshot, + accounts: currentSnapshot.accounts.map((account) => + refreshOperatorAccountCutoverBalanceAudit({ + ...account, + usdWallets: account.usdWallets.map((wallet) => ({ + ...wallet, + balance: balanceCache.get(wallet.id) ?? wallet.balance, + })), + usdtWallets: account.usdtWallets.map((wallet) => ({ + ...wallet, + balance: balanceCache.get(wallet.id) ?? wallet.balance, + })), + }), + ), + treasury: { + ...currentSnapshot.treasury, + accounts: currentSnapshot.treasury.accounts.map((account) => ({ + ...account, + usdWallets: account.usdWallets.map((wallet) => ({ + ...wallet, + balance: balanceCache.get(wallet.id) ?? wallet.balance, + })), + usdtWallets: account.usdtWallets.map((wallet) => ({ + ...wallet, + balance: balanceCache.get(wallet.id) ?? wallet.balance, + })), + })), + }, + }) + + const buildSnapshot = async () => { + const config = await migrationsRepo.getConfig() + if (config instanceof Error) throw config + const lookup = + migrationLookup ?? + (config.runId + ? { + cutoverVersion: config.cutoverVersion, + runId: config.runId, + } + : undefined) + const discoveries = lookup + ? await discoverCashWalletCutoverAccounts({ + accountsRepo, + walletsRepo, + }) + : undefined + if (discoveries instanceof Error) throw discoveries + const preflightReport = + lookup && discoveries + ? buildCashWalletCutoverPreflightReport({ + cutoverVersion: lookup.cutoverVersion, + runId: lookup.runId, + discoveries, + }) + : undefined + + const result = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts, + accountsRepo, + walletsRepo, + migrationsRepo, + migrationLookup: lookup, + preflightReport, + discoveredAccounts: discoveries, + treasuryAccountIds, + balanceReadAttempts: BALANCE_READ_ATTEMPTS, + balanceMode: "structural", + getBalanceForWallet: (request) => + readBalanceThrottled({ + walletId: request.walletId, + currency: request.currency ?? WalletCurrency.Usd, + }), + }) + registerSnapshotWallets(result) + return result + } + + const snapshot = async (force: boolean) => { + const now = Date.now() + if (!force && cache && now - cache.cachedAt < args["snapshot-ttl-ms"]) { + return cache.snapshot + } + if (pending) return pending + + pending = buildSnapshot() + .then((result) => { + cache = { snapshot: result, cachedAt: Date.now() } + return result + }) + .finally(() => { + pending = undefined + }) + return pending + } + + const app = express() + app.get("/", (_req, res) => res.type("html").send(html)) + app.get("/api/snapshot", async (req, res) => { + try { + res.json(await snapshot(req.query.refresh === "1")) + } catch (error) { + baseLogger.error({ error }, "Cash wallet cutover dashboard snapshot failed") + res.status(500).json({ + error: error instanceof Error ? error.message : String(error), + }) + } + }) + app.get("/api/balances", async (req, res) => { + try { + await snapshot(false) + const rawWalletIds = + typeof req.query.walletIds === "string" ? req.query.walletIds : "" + const requestedWalletIds = rawWalletIds + ? rawWalletIds + .split(",") + .map((walletId) => walletId.trim()) + .filter(Boolean) + : Array.from(walletCurrencies.keys()) + const force = req.query.refresh === "1" + + for (const rawWalletId of requestedWalletIds) { + const walletId = rawWalletId as WalletId + const currency = walletCurrencies.get(walletId) + if (!currency) continue + enqueueBalance({ walletId, currency, force }) + } + + const balances: Record = {} + for (const rawWalletId of requestedWalletIds) { + const walletId = rawWalletId as WalletId + const cached = balanceCache.get(walletId) + if (cached) balances[walletId] = cached + } + + res.json({ + balances, + queue: { + pending: balanceQueue.length, + active: activeBalanceId, + }, + }) + } catch (error) { + baseLogger.error({ error }, "Cash wallet cutover dashboard balance refresh failed") + res.status(500).json({ + error: error instanceof Error ? error.message : String(error), + }) + } + }) + app.get("/api/balance-status", async (_req, res) => { + const balances = Array.from(balanceCache.values()) + res.json({ + known: walletCurrencies.size, + cached: balances.length, + fresh: balances.filter((balance) => balance.status === "fresh").length, + errors: balances.filter((balance) => balance.status === "error").length, + loading: balances.filter((balance) => balance.status === "loading").length, + queue: { + pending: balanceQueue.length, + active: activeBalanceId, + }, + }) + }) + app.get("/api/export.csv", async (_req, res) => { + try { + const currentSnapshot = await snapshot(false) + const csv = formatCashWalletCutoverOperatorSnapshotCsv( + snapshotWithCachedBalances(currentSnapshot), + ) + const runId = currentSnapshot.cutover.runId ?? "unknown-run" + res + .type("text/csv") + .attachment(`cash-wallet-cutover-${runId}-${Date.now()}.csv`) + .send(csv) + } catch (error) { + baseLogger.error({ error }, "Cash wallet cutover dashboard CSV export failed") + res.status(500).json({ + error: error instanceof Error ? error.message : String(error), + }) + } + }) + + const server = http.createServer(app) + server.listen(args.port, "127.0.0.1", () => { + baseLogger.info( + { port: args.port, accounts: manifestAccounts.length }, + "Cash wallet cutover dashboard listening", + ) + console.log(`Cash wallet cutover dashboard: http://localhost:${args.port}`) + }) +} + +start().catch((error) => { + baseLogger.error({ error }, "Cash wallet cutover dashboard failed") + process.exit(1) +}) diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index 3bd9a173a..c417547e7 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -4,6 +4,7 @@ import yargs from "yargs" import { hideBin } from "yargs/helpers" import { CashWalletCutover } from "@app" +import { addWalletIfNonexistent } from "@app/accounts" import { setupMongoConnection } from "@services/mongodb" import { AccountsRepository, @@ -14,6 +15,10 @@ import { baseLogger } from "@services/logger" const args = yargs(hideBin(process.argv)) .command("preview", "discover accounts and print the migration plan without writes") + .command( + "provision-usdt-wallets", + "create missing destination USDT wallets before preparing migrations", + ) .command("prepare", "discover accounts and upsert migration records") .command("start", "mark a prepared cutover run in progress") .command("run-batch", "run one locked migration worker batch") @@ -25,6 +30,10 @@ const args = yargs(hideBin(process.argv)) .option("operator", { type: "string", default: "unknown" }) .option("worker-id", { type: "string", default: `worker-${process.pid}` }) .option("limit", { type: "number", default: 25 }) + .option("step-delay-ms", { type: "number", default: 0 }) + .option("provision-limit", { type: "number" }) + .option("provision-delay-ms", { type: "number", default: 12_500 }) + .option("dry-run", { type: "boolean", default: false }) .option("lock-stale-seconds", { type: "number", default: 300 }) .option("configPath", { type: "string", demandOption: true }) .parseSync() @@ -51,6 +60,28 @@ const run = async () => { return } + case "provision-usdt-wallets": { + const result = await CashWalletCutover.provisionPrimaryCashWalletUsdtWallets({ + cutoverVersion, + runId, + accountsRepo: AccountsRepository(), + walletsRepo: WalletsRepository(), + migrationsRepo: repository, + addWalletIfNonexistent, + provisionLimit: args["provision-limit"], + provisionDelayMs: args["provision-delay-ms"], + dryRun: args["dry-run"], + }) + if (result instanceof Error) throw result + toJson(result) + if (result.failed.length > 0) { + throw new Error( + `Failed to provision ${result.failed.length} destination USDT wallet(s)`, + ) + } + return + } + case "prepare": { const result = await CashWalletCutover.preparePrimaryCashWalletCutover({ cutoverVersion, @@ -82,6 +113,7 @@ const run = async () => { runId, workerId: args["worker-id"], limit: args.limit, + stepDelayMs: args["step-delay-ms"], lockStaleBefore: new Date(Date.now() - args["lock-stale-seconds"] * 1000), migrationsRepo: repository, }) diff --git a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts new file mode 100644 index 000000000..78ca82170 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts @@ -0,0 +1,782 @@ +import { + buildCashWalletCutoverOperatorSnapshot, + formatCashWalletCutoverOperatorSnapshotCsv, + parseCashWalletCutoverOperatorManifest, +} from "@app/cash-wallet-cutover/operator-dashboard" +import { USDAmount, USDTAmount, WalletCurrency } from "@domain/shared" +import { WalletType } from "@domain/wallets" + +const account = ({ + id, + defaultWalletId, + uuid, + role, +}: { + id: AccountId + defaultWalletId: WalletId + uuid?: AccountUuid + role?: string +}): Account => + ({ + id, + uuid, + defaultWalletId, + role, + }) as Account + +const wallet = ({ + id, + accountId, + currency, +}: { + id: WalletId + accountId: AccountId + currency: WalletCurrency +}): Wallet => ({ + id, + accountId, + currency, + type: WalletType.Checking, + onChainAddressIdentifiers: [], + onChainAddresses: () => [], + lnurlp: "" as Lnurl, +}) + +describe("cash wallet cutover operator dashboard", () => { + it("parses both generated manifest shapes", () => { + expect( + parseCashWalletCutoverOperatorManifest({ + runId: "eng345usd", + accounts: [ + { + index: 1, + phone: "+16509940000", + username: "eng345usd01", + accountId: "account-1", + usdWalletId: "usd-1", + usdtWalletId: "usdt-1", + }, + ], + }), + ).toEqual([ + { + batchRunId: "eng345usd", + index: 1, + phone: "+16509940000", + username: "eng345usd01", + accountId: "account-1", + expectedUsdWalletId: "usd-1", + expectedUsdtWalletId: "usdt-1", + }, + ]) + + expect( + parseCashWalletCutoverOperatorManifest({ + runId: "eng345usdonly", + created: [ + { + index: 1, + phone: "+16509941000", + accountId: "account-2", + usdWalletId: "usd-2", + }, + ], + }), + ).toEqual([ + { + batchRunId: "eng345usdonly", + index: 1, + phone: "+16509941000", + accountId: "account-2", + expectedUsdWalletId: "usd-2", + }, + ]) + }) + + it("formats the full operator snapshot as escaped account-level CSV", () => { + const csv = formatCashWalletCutoverOperatorSnapshotCsv({ + generatedAt: "2026-05-28T20:00:00.000Z", + cutover: { + state: "in_progress" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: "2026-05-28T19:59:00.000Z", + }, + summary: { + accounts: 1, + wallets: { + current: 2, + target: 2, + usd: 1, + usdt: 1, + missingUsdt: 0, + }, + fundedUsdOnlyAccounts: 0, + usdTotalCents: 123, + usdtTotalMicros: 456_000, + anomalies: 1, + canStart: false, + blockers: 0, + watchlistAccounts: 1, + migrationStatuses: { complete: 1 }, + }, + accounts: [ + { + batchRunId: "batch,one", + index: 1, + phone: "+16509940000", + username: 'quoted"user', + accountId: "account-1" as AccountId, + accountUuid: "uuid-1" as AccountUuid, + expectedUsdWalletId: "usd-1" as WalletId, + expectedUsdtWalletId: "usdt-1" as WalletId, + watchlisted: true, + defaultWalletId: "usd-1" as WalletId, + defaultWalletCurrency: WalletCurrency.Usd, + walletCount: 2, + usdWallets: [ + { + id: "usd-1" as WalletId, + currency: WalletCurrency.Usd, + expected: true, + balance: { + currency: WalletCurrency.Usd, + display: "$1.23", + minorUnits: "123", + minorUnitsNumber: 123, + status: "fresh", + }, + }, + ], + usdtWallets: [ + { + id: "usdt-1" as WalletId, + currency: WalletCurrency.Usdt, + expected: true, + balance: { + currency: WalletCurrency.Usdt, + display: "0.46 USDT", + minorUnits: "456000", + minorUnitsNumber: 456000, + status: "fresh", + }, + }, + ], + migrationStatus: "complete", + migrationUpdatedAt: "2026-05-28T20:00:00.000Z", + cutoverBalanceAudit: { + status: "verified", + sourceUsdCents: 123, + expectedMinimumUsdtMicros: 1_230_000, + destinationStartingBalanceUsdtMicros: 0, + currentDestinationBalanceUsdtMicros: 1_240_000, + finalDeltaUsdtMicros: 1_240_000, + roundingSubsidyUsdtMicros: 10_000, + shortfallUsdtMicros: 0, + }, + anomalies: ["manual,review"], + }, + ], + }) + + expect(csv.split("\n")[0]).toBe( + [ + "generatedAt", + "cutoverState", + "cutoverVersion", + "cutoverRunId", + "cutoverUpdatedAt", + "watchlisted", + "batchRunId", + "index", + "phone", + "username", + "accountId", + "accountUuid", + "defaultWalletId", + "defaultWalletCurrency", + "expectedUsdWalletId", + "expectedUsdtWalletId", + "walletCount", + "usdWalletIds", + "usdBalanceDisplays", + "usdBalanceMinorUnits", + "usdBalanceStatuses", + "usdtWalletIds", + "usdtBalanceDisplays", + "usdtBalanceMinorUnits", + "usdtBalanceStatuses", + "migrationStatus", + "migrationUpdatedAt", + "cutoverBalanceAudit", + "anomalies", + ].join(","), + ) + expect(csv).toContain('"batch,one"') + expect(csv).toContain('"quoted""user"') + expect(csv).toContain('"manual,review"') + expect(csv).toContain("usd-1") + expect(csv).toContain("usdt-1") + expect(csv).toContain("roundingSubsidyUsdtMicros=10000") + }) + + it("summarizes raw wallets, balances, migrations, and anomalies", async () => { + const usdTenCents = USDAmount.cents(10n) + const usdtTwentyFiveCents = USDTAmount.smallestUnits(250_000n) + if (usdTenCents instanceof Error) throw usdTenCents + if (usdtTwentyFiveCents instanceof Error) throw usdtTwentyFiveCents + + const accounts = new Map([ + [ + "account-1", + account({ + id: "account-1" as AccountId, + uuid: "uuid-1" as AccountUuid, + defaultWalletId: "usd-1" as WalletId, + }), + ], + [ + "account-2", + account({ + id: "account-2" as AccountId, + uuid: "uuid-2" as AccountUuid, + defaultWalletId: "usd-2" as WalletId, + }), + ], + ]) + + const wallets = new Map([ + [ + "account-1", + [ + wallet({ + id: "usd-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "usdt-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usdt, + }), + ], + ], + [ + "account-2", + [ + wallet({ + id: "usd-2" as WalletId, + accountId: "account-2" as AccountId, + currency: WalletCurrency.Usd, + }), + ], + ], + ]) + + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [ + { + batchRunId: "batch", + index: 1, + phone: "+16509940000", + accountId: "account-1" as AccountId, + }, + { + batchRunId: "batch", + index: 2, + phone: "+16509941000", + accountId: "account-2" as AccountId, + }, + ], + accountsRepo: { + findById: jest.fn(async (id: AccountId) => accounts.get(id) as Account), + }, + walletsRepo: { + listByAccountId: jest.fn(async (id: AccountId) => wallets.get(id) ?? []), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "in_progress" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn( + async ({ accountId }: { accountId: AccountId }) => + accountId === "account-1" + ? { + id: "migration-1", + accountId, + legacyUsdWalletId: "usd-1" as WalletId, + destinationUsdtWalletId: "usdt-1" as WalletId, + cutoverVersion: 7, + runId: "run-7", + status: "complete" as CashWalletMigrationStatus, + idempotencyKey: "key", + attempts: 1, + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + } + : null, + ), + }, + getBalanceForWallet: jest.fn(async ({ walletId }: { walletId: WalletId }) => + walletId === "usdt-1" ? usdtTwentyFiveCents : usdTenCents, + ), + preflightReport: { + cutoverVersion: 7, + runId: "run-7", + totalAccounts: 101, + migrationCandidates: 81, + alreadyUsdt: 10, + residualLegacyUsd: 0, + blockers: 10, + blockerAccounts: [], + canStart: false, + }, + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(snapshot.preflight).toMatchObject({ + totalAccounts: 101, + migrationCandidates: 81, + blockers: 10, + canStart: false, + }) + expect(snapshot.summary.accounts).toBe(2) + expect(snapshot.summary.wallets.current).toBe(3) + expect(snapshot.summary.wallets.target).toBe(4) + expect(snapshot.summary.wallets.missingUsdt).toBe(1) + expect(snapshot.summary.canStart).toBe(false) + expect(snapshot.summary.blockers).toBe(1) + expect(snapshot.summary.fundedUsdOnlyAccounts).toBe(1) + expect(snapshot.summary.usdTotalCents).toBe(20) + expect(snapshot.summary.usdtTotalMicros).toBe(250000) + expect(snapshot.summary.migrationStatuses).toEqual({ complete: 1, none: 1 }) + expect(snapshot.accounts[1].anomalies).toContain("missing_usdt") + }) + + it("reports completed migration final balance audit fields", async () => { + const zeroUsd = USDAmount.cents(0n) + const finalUsdt = USDTAmount.smallestUnits(108_000n) + if (zeroUsd instanceof Error) throw zeroUsd + if (finalUsdt instanceof Error) throw finalUsdt + + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [ + { + index: 1, + accountId: "account-1" as AccountId, + expectedUsdWalletId: "usd-1" as WalletId, + expectedUsdtWalletId: "usdt-1" as WalletId, + }, + ], + accountsRepo: { + findById: jest.fn(async () => + account({ + id: "account-1" as AccountId, + defaultWalletId: "usdt-1" as WalletId, + }), + ), + }, + walletsRepo: { + listByAccountId: jest.fn(async () => [ + wallet({ + id: "usd-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "usdt-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usdt, + }), + ]), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "complete" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn(async () => ({ + id: "migration-1", + accountId: "account-1" as AccountId, + legacyUsdWalletId: "usd-1" as WalletId, + destinationUsdtWalletId: "usdt-1" as WalletId, + cutoverVersion: 7, + runId: "run-7", + status: "complete" as CashWalletMigrationStatus, + sourceBalanceUsdCents: "10", + destinationAmountUsdtMicros: "100000", + destinationStartingBalanceUsdtMicros: "0", + feeAmountUsdtMicros: "2000", + feeAmountUsdCents: "1", + idempotencyKey: "key", + attempts: 1, + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + }, + getBalanceForWallet: jest.fn(async ({ currency }: { currency?: WalletCurrency }) => + currency === WalletCurrency.Usdt ? finalUsdt : zeroUsd, + ), + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(snapshot.accounts[0].cutoverBalanceAudit).toEqual({ + status: "verified", + sourceUsdCents: 10, + expectedMinimumUsdtMicros: 100_000, + destinationStartingBalanceUsdtMicros: 0, + currentDestinationBalanceUsdtMicros: 108_000, + finalDeltaUsdtMicros: 108_000, + roundingSubsidyUsdtMicros: 8_000, + shortfallUsdtMicros: 0, + }) + }) + + it("includes funder balances in reconciliation without adding migration rows", async () => { + const customerUsd = USDAmount.cents(452n) + const customerUsdt = USDTAmount.smallestUnits(45_200_00n) + const funderUsd = USDAmount.cents(418n) + const funderUsdt = USDTAmount.smallestUnits(9_900_000n) + if (customerUsd instanceof Error) throw customerUsd + if (customerUsdt instanceof Error) throw customerUsdt + if (funderUsd instanceof Error) throw funderUsd + if (funderUsdt instanceof Error) throw funderUsdt + + const accounts = new Map([ + [ + "customer-account", + account({ + id: "customer-account" as AccountId, + defaultWalletId: "customer-usdt" as WalletId, + }), + ], + [ + "funder-account", + account({ + id: "funder-account" as AccountId, + defaultWalletId: "funder-usd" as WalletId, + role: "funder", + }), + ], + ]) + + const wallets = new Map([ + [ + "customer-account", + [ + wallet({ + id: "customer-usd" as WalletId, + accountId: "customer-account" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "customer-usdt" as WalletId, + accountId: "customer-account" as AccountId, + currency: WalletCurrency.Usdt, + }), + ], + ], + [ + "funder-account", + [ + wallet({ + id: "funder-usd" as WalletId, + accountId: "funder-account" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "funder-usdt" as WalletId, + accountId: "funder-account" as AccountId, + currency: WalletCurrency.Usdt, + }), + ], + ], + ]) + + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [], + discoveredAccounts: [ + { + status: "usdt_default", + accountId: "customer-account" as AccountId, + legacyUsdWalletId: "customer-usd" as WalletId, + destinationUsdtWalletId: "customer-usdt" as WalletId, + previousDefaultWalletId: "customer-usd" as WalletId, + }, + ], + treasuryAccountIds: ["funder-account" as AccountId], + accountsRepo: { + findById: jest.fn(async (id: AccountId) => accounts.get(id) as Account), + }, + walletsRepo: { + listByAccountId: jest.fn(async (id: AccountId) => wallets.get(id) ?? []), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "complete" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn(async () => null), + }, + getBalanceForWallet: jest.fn(async ({ walletId }: { walletId: WalletId }) => { + if (walletId === "customer-usd") return customerUsd + if (walletId === "customer-usdt") return customerUsdt + if (walletId === "funder-usd") return funderUsd + return funderUsdt + }), + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(snapshot.accounts.map((row) => row.accountId)).toEqual([ + "customer-account", + ]) + expect(snapshot.treasury.accounts.map((row) => row.accountId)).toEqual([ + "funder-account", + ]) + expect(snapshot.summary.usdTotalCents).toBe(452) + expect(snapshot.summary.usdtTotalMicros).toBe(4_520_000) + expect(snapshot.treasury.summary.usdTotalCents).toBe(418) + expect(snapshot.treasury.summary.usdtTotalMicros).toBe(9_900_000) + expect(snapshot.reconciliation.customerTotalCents).toBe(904) + expect(snapshot.reconciliation.treasuryTotalCents).toBe(1_408) + expect(snapshot.reconciliation.systemTotalCents).toBe(2_312) + }) + + it("uses global discoveries as dashboard rows while highlighting manifest accounts", async () => { + const zeroUsd = USDAmount.cents(0n) + const zeroUsdt = USDTAmount.smallestUnits(0n) + if (zeroUsd instanceof Error) throw zeroUsd + if (zeroUsdt instanceof Error) throw zeroUsdt + + const accounts = new Map([ + [ + "watchlist-account", + account({ + id: "watchlist-account" as AccountId, + uuid: "watchlist-uuid" as AccountUuid, + defaultWalletId: "watchlist-usd" as WalletId, + }), + ], + [ + "global-account", + account({ + id: "global-account" as AccountId, + uuid: "global-uuid" as AccountUuid, + defaultWalletId: "global-usd" as WalletId, + }), + ], + ]) + + const wallets = new Map([ + [ + "watchlist-account", + [ + wallet({ + id: "watchlist-usd" as WalletId, + accountId: "watchlist-account" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "watchlist-usdt" as WalletId, + accountId: "watchlist-account" as AccountId, + currency: WalletCurrency.Usdt, + }), + ], + ], + [ + "global-account", + [ + wallet({ + id: "global-usd" as WalletId, + accountId: "global-account" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "global-usdt" as WalletId, + accountId: "global-account" as AccountId, + currency: WalletCurrency.Usdt, + }), + ], + ], + ]) + + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [ + { + batchRunId: "batch", + index: 1, + phone: "+16509940000", + accountId: "watchlist-account" as AccountId, + expectedUsdWalletId: "watchlist-usd" as WalletId, + expectedUsdtWalletId: "watchlist-usdt" as WalletId, + }, + ], + discoveredAccounts: [ + { + status: "legacy_default", + accountId: "watchlist-account" as AccountId, + accountUuid: "watchlist-uuid" as AccountUuid, + legacyUsdWalletId: "watchlist-usd" as WalletId, + destinationUsdtWalletId: "watchlist-usdt" as WalletId, + previousDefaultWalletId: "watchlist-usd" as WalletId, + }, + { + status: "legacy_default", + accountId: "global-account" as AccountId, + accountUuid: "global-uuid" as AccountUuid, + legacyUsdWalletId: "global-usd" as WalletId, + destinationUsdtWalletId: "global-usdt" as WalletId, + previousDefaultWalletId: "global-usd" as WalletId, + }, + ], + accountsRepo: { + findById: jest.fn(async (id: AccountId) => accounts.get(id) as Account), + }, + walletsRepo: { + listByAccountId: jest.fn(async (id: AccountId) => wallets.get(id) ?? []), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "pre" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn(async () => null), + }, + getBalanceForWallet: jest.fn(async ({ currency }: { currency?: WalletCurrency }) => + currency === WalletCurrency.Usdt ? zeroUsdt : zeroUsd, + ), + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(snapshot.summary.accounts).toBe(2) + expect(snapshot.summary.watchlistAccounts).toBe(1) + expect(snapshot.accounts.map((row) => row.accountId)).toEqual([ + "watchlist-account", + "global-account", + ]) + expect(snapshot.accounts[0].watchlisted).toBe(true) + expect(snapshot.accounts[0].phone).toBe("+16509940000") + expect(snapshot.accounts[1].watchlisted).toBe(false) + expect(snapshot.accounts[1].expectedUsdWalletId).toBe("global-usd") + expect(snapshot.accounts[1].expectedUsdtWalletId).toBe("global-usdt") + expect(snapshot.accounts[1].anomalies).not.toContain("unexpected_wallet_id") + }) + + it("retries transient balance read errors before marking a wallet anomalous", async () => { + const balance = USDAmount.cents(10n) + if (balance instanceof Error) throw balance + + const getBalanceForWallet = jest + .fn() + .mockResolvedValueOnce(new Error("temporary ibex failure")) + .mockResolvedValueOnce(balance) + + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [ + { + index: 1, + phone: "+16509941000", + accountId: "account-1" as AccountId, + }, + ], + accountsRepo: { + findById: jest.fn(async () => + account({ + id: "account-1" as AccountId, + defaultWalletId: "usd-1" as WalletId, + }), + ), + }, + walletsRepo: { + listByAccountId: jest.fn(async () => [ + wallet({ + id: "usd-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usd, + }), + ]), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "complete" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn(async () => null), + }, + getBalanceForWallet, + balanceReadAttempts: 2, + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(getBalanceForWallet).toHaveBeenCalledTimes(2) + expect(snapshot.accounts[0].usdWallets[0].balance.display).toBe("$0.10") + expect(snapshot.accounts[0].anomalies).toEqual(["missing_usdt"]) + }) + + it("builds a structural snapshot without reading wallet balances", async () => { + const getBalanceForWallet = jest.fn() + + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [ + { + index: 1, + phone: "+16509941000", + accountId: "account-1" as AccountId, + expectedUsdWalletId: "usd-1" as WalletId, + }, + ], + accountsRepo: { + findById: jest.fn(async () => + account({ + id: "account-1" as AccountId, + defaultWalletId: "usd-1" as WalletId, + }), + ), + }, + walletsRepo: { + listByAccountId: jest.fn(async () => [ + wallet({ + id: "usd-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usd, + }), + ]), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "in_progress" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn(async () => null), + }, + getBalanceForWallet, + balanceMode: "structural", + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(getBalanceForWallet).not.toHaveBeenCalled() + expect(snapshot.summary.wallets.current).toBe(1) + expect(snapshot.summary.usdTotalCents).toBe(0) + expect(snapshot.summary.fundedUsdOnlyAccounts).toBe(0) + expect(snapshot.accounts[0].usdWallets[0]).toMatchObject({ + id: "usd-1", + balance: { + status: "loading", + display: "loading", + minorUnitsNumber: 0, + }, + }) + expect(snapshot.accounts[0].anomalies).toEqual(["missing_usdt"]) + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts b/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts new file mode 100644 index 000000000..94733f022 --- /dev/null +++ b/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts @@ -0,0 +1,169 @@ +import { WalletCurrency } from "@domain/shared" +import { WalletType } from "@domain/wallets" + +import { InvalidCashWalletCutoverStateTransitionError } from "@app/cash-wallet-cutover/errors" +import { provisionPrimaryCashWalletUsdtWallets } from "@app/cash-wallet-cutover/provision-usdt-wallets" + +const account = (id: AccountId, defaultWalletId: WalletId): Account => + ({ + id, + uuid: `${id}-uuid` as AccountUuid, + defaultWalletId, + }) as Account + +const wallet = (accountId: AccountId, id: WalletId, currency: WalletCurrency): Wallet => + ({ + id, + accountId, + type: WalletType.Checking, + currency, + onChainAddressIdentifiers: [], + onChainAddresses: () => [], + lnurlp: "lnurl" as Lnurl, + }) as Wallet + +async function* unlockedAccounts(accounts: Account[]): AsyncGenerator { + for (const account of accounts) yield account +} + +const config = (state: CashWalletCutoverState): CashWalletCutoverConfig => ({ + state, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-20T00:00:00Z"), +}) + +describe("provision primary cash wallet USDT wallets", () => { + it("provisions missing USDT wallets without changing existing defaults", async () => { + const missingUsdtAccount = account( + "missing-account" as AccountId, + "missing-account-usd" as WalletId, + ) + const migrationCandidate = account( + "candidate-account" as AccountId, + "candidate-account-usd" as WalletId, + ) + const alreadyUsdtAccount = account( + "already-usdt-account" as AccountId, + "already-usdt-account-usdt" as WalletId, + ) + const accounts = [missingUsdtAccount, migrationCandidate, alreadyUsdtAccount] + const walletsByAccountId = new Map([ + [ + missingUsdtAccount.id, + [ + wallet( + missingUsdtAccount.id, + "missing-account-usd" as WalletId, + WalletCurrency.Usd, + ), + ], + ], + [ + migrationCandidate.id, + [ + wallet( + migrationCandidate.id, + "candidate-account-usd" as WalletId, + WalletCurrency.Usd, + ), + wallet( + migrationCandidate.id, + "candidate-account-usdt" as WalletId, + WalletCurrency.Usdt, + ), + ], + ], + [ + alreadyUsdtAccount.id, + [ + wallet( + alreadyUsdtAccount.id, + "already-usdt-account-usd" as WalletId, + WalletCurrency.Usd, + ), + wallet( + alreadyUsdtAccount.id, + "already-usdt-account-usdt" as WalletId, + WalletCurrency.Usdt, + ), + ], + ], + ]) + const provisionedWallet = wallet( + missingUsdtAccount.id, + "missing-account-usdt" as WalletId, + WalletCurrency.Usdt, + ) + const addWalletIfNonexistent = jest.fn(async () => { + walletsByAccountId.set(missingUsdtAccount.id, [ + ...(walletsByAccountId.get(missingUsdtAccount.id) ?? []), + provisionedWallet, + ]) + return provisionedWallet + }) + + const result = await provisionPrimaryCashWalletUsdtWallets({ + cutoverVersion: 7, + runId: "run-7", + accountsRepo: { listUnlockedAccounts: () => unlockedAccounts(accounts) }, + walletsRepo: { + listByAccountId: jest.fn( + async (accountId: AccountId) => walletsByAccountId.get(accountId) ?? [], + ), + }, + migrationsRepo: { getConfig: jest.fn(async () => config("pre")) }, + addWalletIfNonexistent, + sleep: jest.fn(), + }) + + expect(result).toMatchObject({ + before: { + totalAccounts: 3, + migrationCandidates: 1, + alreadyUsdt: 1, + blockers: 1, + canStart: false, + }, + eligible: 1, + provisioned: [ + { + accountId: "missing-account", + walletId: "missing-account-usdt", + }, + ], + failed: [], + after: { + totalAccounts: 3, + migrationCandidates: 2, + alreadyUsdt: 1, + blockers: 0, + canStart: true, + }, + }) + expect(addWalletIfNonexistent).toHaveBeenCalledTimes(1) + expect(addWalletIfNonexistent).toHaveBeenCalledWith({ + accountId: missingUsdtAccount.id, + type: WalletType.Checking, + currency: WalletCurrency.Usdt, + }) + expect(missingUsdtAccount.defaultWalletId).toBe("missing-account-usd") + expect(alreadyUsdtAccount.defaultWalletId).toBe("already-usdt-account-usdt") + }) + + it("does not provision after the cutover has started", async () => { + const addWalletIfNonexistent = jest.fn() + + const result = await provisionPrimaryCashWalletUsdtWallets({ + cutoverVersion: 7, + runId: "run-7", + accountsRepo: { listUnlockedAccounts: () => unlockedAccounts([]) }, + walletsRepo: { listByAccountId: jest.fn() }, + migrationsRepo: { getConfig: jest.fn(async () => config("in_progress")) }, + addWalletIfNonexistent, + }) + + expect(result).toBeInstanceOf(InvalidCashWalletCutoverStateTransitionError) + expect(addWalletIfNonexistent).not.toHaveBeenCalled() + }) +}) diff --git a/test/flash/unit/app/cash-wallet-cutover/runner.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runner.spec.ts index 6987c3873..eb01920f3 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runner.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runner.spec.ts @@ -123,4 +123,41 @@ describe("cash wallet migration batch runner", () => { }) expect(migrationsRepo.releaseMigrationLock).not.toHaveBeenCalled() }) + + it("waits between attempted migrations when step delay is configured", async () => { + const runnable = [ + migration("provisioned", "migration-1"), + migration("provisioned", "migration-2"), + migration("provisioned", "migration-3"), + ] + const migrationsRepo = { + listRunnableMigrations: jest.fn(async () => runnable), + acquireMigrationLock: jest.fn(async (args: { id: string }) => + migration("provisioned", args.id), + ), + markMigrationFailed: jest.fn(), + releaseMigrationLock: jest.fn(async () => migration("balance_read")), + } + const executor = jest.fn(async (locked: CashWalletMigration) => + migration("balance_read", locked.id), + ) + const sleep = jest.fn(async () => undefined) + + const result = await runCashWalletMigrationBatch({ + cutoverVersion: 7, + runId: "run-7", + workerId: "worker-1", + limit: 3, + lockStaleBefore: new Date("2026-05-20T15:00:00Z"), + migrationsRepo, + executor, + stepDelayMs: 1_000, + sleep, + }) + + expect(result).toEqual({ attempted: 3, advanced: 3, failed: 0, skipped: 0 }) + expect(sleep).toHaveBeenCalledTimes(2) + expect(sleep).toHaveBeenNthCalledWith(1, 1_000) + expect(sleep).toHaveBeenNthCalledWith(2, 1_000) + }) }) diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts index 45e793e60..d02527b9b 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -48,6 +48,10 @@ const migration = (patch: Partial = {}): CashWalletMigratio }) describe("cash wallet migration runtime services", () => { + beforeEach(() => { + jest.clearAllMocks() + }) + it("reads source USD balances as cents", async () => { const deps = { getBalanceForWallet: jest.fn(async () => USDAmount.cents("1234")), @@ -123,6 +127,29 @@ describe("cash wallet migration runtime services", () => { }) }) + it("creates amount destination invoices in exact USDT micros through IBEX", async () => { + jest.mocked(Ibex.addInvoice).mockResolvedValue(ibexAddInvoiceResponse as never) + + const services = createCashWalletMigrationRuntimeServices() + + const result = await services.invoiceService.createInvoice({ + recipientWalletId: "usdt-wallet-id" as WalletId, + amount: "4711", + memo: "cash-wallet-cutover:run-7:migration-id:fee-reimbursement", + }) + + expect(result).toMatchObject({ + paymentRequest: ibexAddInvoiceResponse.invoice.bolt11, + }) + const args = jest.mocked(Ibex.addInvoice).mock.calls[0][0]! + expect(args.accountId).toBe("usdt-wallet-id") + expect(args.memo).toBe("cash-wallet-cutover:run-7:migration-id:fee-reimbursement") + expect(args.expiration).toBe(900) + expect(args.amount).toBeInstanceOf(USDTAmount) + expect((args.amount as USDTAmount).asSmallestUnits()).toBe("4711") + expect((args.amount as USDTAmount).toIbex()).toBe(0.004711) + }) + it("extracts the IBEX transaction id after paying an invoice with a sender-side USD cap", async () => { const deps = { payInvoice: jest.fn(async () => ({ diff --git a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts index 0d27c14ae..fb4699724 100644 --- a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts @@ -475,7 +475,7 @@ describe("cash wallet migration worker checkpoints", () => { expect(migrationsRepo.transitionMigration).not.toHaveBeenCalled() }) - it("creates a fee reimbursement invoice on the destination wallet for the exact USDT shortfall", async () => { + it("creates a fee reimbursement invoice rounded up to USD-cent USDT micros", async () => { const migrationsRepo = { transitionMigration: jest.fn(async () => ({ ...migration("fee_reimbursement_invoice_created"), @@ -509,7 +509,7 @@ describe("cash wallet migration worker checkpoints", () => { }) expect(invoiceService.createInvoice).toHaveBeenCalledWith({ recipientWalletId: "usdt-wallet-id", - amount: "70001", + amount: "80000", memo: "cash-wallet-cutover:run-7:migration-id:fee-reimbursement", }) expect(migrationsRepo.transitionMigration).toHaveBeenCalledWith({ diff --git a/test/flash/unit/app/wallets/usd-wallet-amount.spec.ts b/test/flash/unit/app/wallets/usd-wallet-amount.spec.ts index 905c76477..3fbeb8b17 100644 --- a/test/flash/unit/app/wallets/usd-wallet-amount.spec.ts +++ b/test/flash/unit/app/wallets/usd-wallet-amount.spec.ts @@ -19,6 +19,15 @@ describe("usdWalletAmountFromInput", () => { expect((amount as USDTAmount).toIbex()).toBe(194.46) }) + it("converts small USDT cent inputs to micro-USDT", () => { + const amount = usdWalletAmountFromInput("30", WalletCurrency.Usdt) + + expect(amount).toBeInstanceOf(USDTAmount) + expect((amount as USDTAmount).asSmallestUnits()).toBe("300000") + expect((amount as USDTAmount).asNumber()).toBe("0.300000") + expect((amount as USDTAmount).toIbex()).toBe(0.3) + }) + it("rejects BTC", () => { const amount = usdWalletAmountFromInput("19446", WalletCurrency.Btc) From aa21785fe9a9a6d98aae5e35f874b9dc8d7d6190 Mon Sep 17 00:00:00 2001 From: Vandana Date: Sun, 31 May 2026 09:04:08 -0700 Subject: [PATCH 2/5] fix(cutover): use ibex oauth credentials in local setup --- .env | 8 +++--- DEV.md | 28 ++++---------------- dev/config/set-overrides.sh | 7 ++--- dev/setup.sh | 26 ++++++++++-------- docker-compose.yml | 5 ++-- src/services/ibex/client.ts | 53 ++++++++++++++++++++----------------- 6 files changed, 58 insertions(+), 69 deletions(-) diff --git a/.env b/.env index 44699ef99..59076eba4 100644 --- a/.env +++ b/.env @@ -123,8 +123,8 @@ export ERPNEXT_JWT_SECRET="not-so-secret" COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml:docker-compose.local.yml -# Ibex API (used by docker compose; app reads from yaml config) -export IBEX_URL="https://api-sandbox.poweredbyibex.io" -export IBEX_EMAIL="" -export IBEX_PASSWORD="" +# Ibex API — app reads from yaml config, not env vars +# Get sandbox credentials from your team lead +# export IBEX_CLIENT_ID="" +# export IBEX_CLIENT_SECRET="" diff --git a/DEV.md b/DEV.md index 0b17a2e58..05f11208c 100644 --- a/DEV.md +++ b/DEV.md @@ -38,30 +38,11 @@ If you prefer to set things up yourself, or if the setup script fails: ### 1. Environment Variables -The project loads environment variables from `.env` (committed) and `.env.local` (git-ignored, for secrets). - -Create `.env.local` with your Ibex sandbox credentials: - -```bash -echo "export IBEX_EMAIL='your-ibex-email'" >> .env.local -echo "export IBEX_PASSWORD='your-ibex-password'" >> .env.local -``` - -If you use direnv, allow it: - -```bash -direnv allow -``` - -If not using direnv, source the env files manually before running commands: - -```bash -source .env && source .env.local -``` +Flash uses YAML config files. Ibex OAuth2 credentials go in local config overrides, not env vars. ### 2. App Config Overrides -Flash uses YAML config files. The base config is at `dev/config/base-config.yaml`. Secrets and local overrides go in `$CONFIG_PATH/dev-overrides.yaml` (default: `~/.config/flash/dev-overrides.yaml`). +The base config is at `dev/config/base-config.yaml`. Secrets and local overrides go in `$CONFIG_PATH/dev-overrides.yaml` (default: `~/.config/flash/dev-overrides.yaml`). **Option A — Run the interactive script:** @@ -74,8 +55,9 @@ Flash uses YAML config files. The base config is at `dev/config/base-config.yaml ```yaml # ~/.config/flash/dev-overrides.yaml ibex: - email: your-ibex-email - password: your-ibex-password + clientId: your-sandbox-client-id + clientSecret: your-sandbox-client-secret + environment: sandbox ``` Additional overrides you might need: diff --git a/dev/config/set-overrides.sh b/dev/config/set-overrides.sh index 18d86797d..d8b0c03aa 100755 --- a/dev/config/set-overrides.sh +++ b/dev/config/set-overrides.sh @@ -7,8 +7,9 @@ mkdir -p "$(dirname "$OUTPUT_FILE")" # Define YAML paths and their descriptions directly in the script declare -a yaml_paths=( - "ibex.email, Email address to Ibex Account" - "ibex.password, Password to Ibex Account" + "ibex.clientId, OAuth2 client ID for the Ibex account" + "ibex.clientSecret, OAuth2 client secret for the Ibex account" + "ibex.environment, Ibex environment: sandbox or production" "ibex.webhook.uri, The URI where Ibex will send payment events" "sendgrid.apiKey, API key to SendGrid email service (from Twilio)" "cashout.email.to, Recipient email address for cashout notifications" @@ -57,4 +58,4 @@ for entry in "${yaml_paths[@]}"; do write_yaml "$path" $value done -echo "YAML file has been written to $OUTPUT_FILE." \ No newline at end of file +echo "YAML file has been written to $OUTPUT_FILE." diff --git a/dev/setup.sh b/dev/setup.sh index 3b001bef7..718b372fa 100755 --- a/dev/setup.sh +++ b/dev/setup.sh @@ -73,24 +73,27 @@ echo "" # ── 5. Configure Ibex credentials ──────────────────── echo "Checking Ibex credentials..." -if [ -f .env.local ] && grep -q "IBEX_PASSWORD" .env.local 2>/dev/null; then +if [ -f .env.local ] && grep -q "IBEX_CLIENT_SECRET" .env.local 2>/dev/null; then info "Ibex credentials found in .env.local" else echo "" - echo "Flash requires Ibex sandbox credentials to connect to the payment backend." + echo "Flash requires Ibex OAuth2 sandbox credentials to connect to the payment backend." echo "If you don't have credentials, ask your team lead." echo "" - read -rp "Ibex email (or press Enter to skip): " IBEX_EMAIL - if [ -n "$IBEX_EMAIL" ]; then - read -rsp "Ibex password: " IBEX_PASSWORD + read -rp "Ibex client ID (or press Enter to skip): " IBEX_CLIENT_ID + if [ -n "$IBEX_CLIENT_ID" ]; then + read -rsp "Ibex client secret: " IBEX_CLIENT_SECRET echo "" + read -rp "Ibex environment [sandbox]: " IBEX_ENVIRONMENT + IBEX_ENVIRONMENT="${IBEX_ENVIRONMENT:-sandbox}" cat > .env.local << EOF -export IBEX_EMAIL='${IBEX_EMAIL}' -export IBEX_PASSWORD='${IBEX_PASSWORD}' +export IBEX_CLIENT_ID='${IBEX_CLIENT_ID}' +export IBEX_CLIENT_SECRET='${IBEX_CLIENT_SECRET}' +export IBEX_ENVIRONMENT='${IBEX_ENVIRONMENT}' EOF info "Credentials saved to .env.local (git-ignored)" else - warn "Skipped — you'll need to create .env.local with IBEX_EMAIL and IBEX_PASSWORD before starting" + warn "Skipped — you'll need to create .env.local with IBEX_CLIENT_ID and IBEX_CLIENT_SECRET before starting" fi fi @@ -109,11 +112,12 @@ else if [ -f .env.local ]; then source .env.local 2>/dev/null || true fi - if [ -n "${IBEX_EMAIL:-}" ] && [ -n "${IBEX_PASSWORD:-}" ]; then + if [ -n "${IBEX_CLIENT_ID:-}" ] && [ -n "${IBEX_CLIENT_SECRET:-}" ]; then cat > "$OVERRIDES" << EOF ibex: - email: ${IBEX_EMAIL} - password: ${IBEX_PASSWORD} + clientId: ${IBEX_CLIENT_ID} + clientSecret: ${IBEX_CLIENT_SECRET} + environment: ${IBEX_ENVIRONMENT:-sandbox} EOF info "Generated $OVERRIDES with Ibex credentials" else diff --git a/docker-compose.yml b/docker-compose.yml index 83fd574a6..690a13c30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,9 +79,8 @@ services: - REDIS_TYPE=standalone - REDIS_0_DNS=redis - REDIS_0_PORT=6378 - - IBEX_URL=${IBEX_URL} - - IBEX_EMAIL=${IBEX_EMAIL} - - IBEX_PASSWORD=${IBEX_PASSWORD} + - IBEX_CLIENT_ID=${IBEX_CLIENT_ID:-} + - IBEX_CLIENT_SECRET=${IBEX_CLIENT_SECRET:-} price-history: image: docker.io/lnflash/price-history:edge # image: us.gcr.io/galoy-org/price-history:edge diff --git a/src/services/ibex/client.ts b/src/services/ibex/client.ts index 8464520e1..cf448778e 100644 --- a/src/services/ibex/client.ts +++ b/src/services/ibex/client.ts @@ -17,6 +17,7 @@ import IbexClient, { PayToALnurlPayResponse201, SendToAddressCopyBodyParam, SendToAddressCopyResponse200, + IbexUrls, } from "ibex-client" import { IbexConfig } from "@config" @@ -55,6 +56,8 @@ const Ibex = new IbexClient( Redis, ) +const IbexUrlConfig = IbexUrls[IbexConfig.environment] + const createAccount = async ( name: string, currencyId: IbexCurrencyId, @@ -262,42 +265,39 @@ const getIbexToken = async (): Promise => { const cached = await Ibex.authentication.storage.getAccessToken() if (typeof cached === "string") return `${cached}` - // The SDK uses a single base URL for all calls, but the sandbox auth domain is separate - const resp = await fetch(`${IbexConfig.url}/auth/signin`, { + const body = new URLSearchParams({ + grant_type: "client_credentials", + client_id: IbexConfig.clientId, + client_secret: IbexConfig.clientSecret, + audience: IbexUrlConfig.audience, + }) + + const resp = await fetch(`${IbexUrlConfig.authDomain}/oauth/token`, { method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ email: IbexConfig.email, password: IbexConfig.password }), + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: body.toString(), }).catch( (err: unknown) => new IbexError(err instanceof Error ? err : new Error(String(err))), ) if (resp instanceof IbexError) return resp if (!resp.ok) { - const body = await resp.text().catch(() => "") - return new IbexError(new Error(`IBEX sign-in failed: ${resp.status} — ${body}`)) + const responseBody = await resp.text().catch(() => "") + return new IbexError( + new Error(`IBEX token request failed: ${resp.status} — ${responseBody}`), + ) } const data = (await resp.json()) as { - accessToken?: string - accessTokenExpiresAt?: number - refreshToken?: string - refreshTokenExpiresAt?: number + access_token?: string + expires_in?: number } - if (!data.accessToken) - return new IbexError(new Error("IBEX sign-in: no access token in response")) + if (!data.access_token) + return new IbexError(new Error("IBEX token request: no access_token in response")) - await Ibex.authentication.storage.setAccessToken( - data.accessToken, - data.accessTokenExpiresAt, - ) - if (data.refreshToken) { - await Ibex.authentication.storage.setRefreshToken( - data.refreshToken, - data.refreshTokenExpiresAt, - ) - } + await Ibex.authentication.storage.setAccessToken(data.access_token, data.expires_in) - return data.accessToken as string + return data.access_token } const ibexFetch = async ( @@ -305,7 +305,7 @@ const ibexFetch = async ( path: string, init: RequestInit = {}, ): Promise => { - const url = `${IbexConfig.url}${path}` + const url = `${IbexUrlConfig.hubUrl}${path}` const resp = await fetch(url, { ...init, headers: { @@ -439,7 +439,10 @@ const getEthereumUsdtOption = async (): Promise const getIbexCurrencyId = async ( currency: WalletCurrency, ): Promise => { - const data = await ibexGet<{ currencies: IbexCurrency[] }>("", "/currency/all") + const token = await getIbexToken() + if (token instanceof IbexError) return token + + const data = await ibexGet<{ currencies: IbexCurrency[] }>(token, "/currency/all") if (data instanceof IbexError) return data const currencyId = data.currencies.find((c) => c.name === currency)?.id if (!currencyId) return new IbexError(new Error(`Currency ${currency} not found`)) From 6eb4ff7cc0d0d1d09f7fded2e5351ead8820e332 Mon Sep 17 00:00:00 2001 From: Vandana Date: Sun, 31 May 2026 16:00:26 -0700 Subject: [PATCH 3/5] fix(cutover): retry wallet provisioning and refresh stale invoices --- src/app/cash-wallet-cutover/handlers.ts | 4 + .../provision-usdt-wallets.ts | 27 ++- .../cash-wallet-cutover/runtime-services.ts | 1 + src/app/cash-wallet-cutover/state-machine.ts | 8 +- src/app/cash-wallet-cutover/worker.ts | 116 ++++++++++-- src/scripts/cash-wallet-cutover.ts | 4 + .../migration-state-machine.spec.ts | 10 ++ .../provision-usdt-wallets.spec.ts | 66 +++++++ .../runtime-services.spec.ts | 12 +- .../app/cash-wallet-cutover/worker.spec.ts | 170 ++++++++++++++++++ 10 files changed, 395 insertions(+), 23 deletions(-) diff --git a/src/app/cash-wallet-cutover/handlers.ts b/src/app/cash-wallet-cutover/handlers.ts index 20d32ff3f..b09756c1d 100644 --- a/src/app/cash-wallet-cutover/handlers.ts +++ b/src/app/cash-wallet-cutover/handlers.ts @@ -114,6 +114,8 @@ export const createCashWalletMigrationStepHandlers = ({ migration, migrationsRepo, paymentService: services.paymentService, + invoiceService: services.invoiceService, + now: services.now, }), balance_move_sending: (migration) => markCashWalletMigrationBalanceMoveSent({ migration, migrationsRepo }), @@ -147,6 +149,8 @@ export const createCashWalletMigrationStepHandlers = ({ migration, migrationsRepo, paymentService: services.paymentService, + invoiceService: services.invoiceService, + now: services.now, treasuryWalletId, }) }, diff --git a/src/app/cash-wallet-cutover/provision-usdt-wallets.ts b/src/app/cash-wallet-cutover/provision-usdt-wallets.ts index 94c4979f7..b66a1be53 100644 --- a/src/app/cash-wallet-cutover/provision-usdt-wallets.ts +++ b/src/app/cash-wallet-cutover/provision-usdt-wallets.ts @@ -49,6 +49,9 @@ const errorMessage = (error: unknown): string => { return String(error) } +const isRateLimitError = (error: unknown): boolean => + errorMessage(error).toLowerCase().includes("too many requests") + export type { ProvisionPrimaryCashWalletUsdtWalletsResult } export const provisionPrimaryCashWalletUsdtWallets = async ({ @@ -60,6 +63,8 @@ export const provisionPrimaryCashWalletUsdtWallets = async ({ addWalletIfNonexistent, provisionLimit, provisionDelayMs = 0, + provisionRetryDelayMs = 60_000, + maxProvisionAttempts = 5, dryRun = false, sleep = defaultSleep, }: { @@ -71,6 +76,8 @@ export const provisionPrimaryCashWalletUsdtWallets = async ({ addWalletIfNonexistent: AddWalletIfNonexistent provisionLimit?: number provisionDelayMs?: number + provisionRetryDelayMs?: number + maxProvisionAttempts?: number dryRun?: boolean sleep?: (delayMs: number) => Promise }): Promise => { @@ -103,11 +110,21 @@ export const provisionPrimaryCashWalletUsdtWallets = async ({ if (!dryRun) { for (const [index, discovery] of eligibleDiscoveries.entries()) { - const wallet = await addWalletIfNonexistent({ - accountId: discovery.accountId, - type: WalletType.Checking, - currency: WalletCurrency.Usdt, - }) + let wallet: Wallet | ApplicationError = new Error("Provisioning was not attempted") + const attempts = Math.max(1, maxProvisionAttempts) + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + wallet = await addWalletIfNonexistent({ + accountId: discovery.accountId, + type: WalletType.Checking, + currency: WalletCurrency.Usdt, + }) + + if (!(wallet instanceof Error)) break + if (!isRateLimitError(wallet) || attempt === attempts) break + + await sleep(provisionRetryDelayMs) + } if (wallet instanceof Error) { failed.push({ accountId: discovery.accountId, error: errorMessage(wallet) }) diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index e854e1cf9..50b2b0f97 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -134,6 +134,7 @@ export const createCashWalletMigrationRuntimeServices = ( }) => noAmountInvoiceForRecipient({ accountId: recipientWalletId, + amount: USDTAmount.ZERO, memo, expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, }).then(ibexInvoiceToDomainInvoice), diff --git a/src/app/cash-wallet-cutover/state-machine.ts b/src/app/cash-wallet-cutover/state-machine.ts index 6bd152b8d..9a950e065 100644 --- a/src/app/cash-wallet-cutover/state-machine.ts +++ b/src/app/cash-wallet-cutover/state-machine.ts @@ -7,7 +7,12 @@ const transitions: Partial< started: ["provisioned", "failed"], provisioned: ["balance_read", "failed", "skipped_already_migrated"], balance_read: ["invoice_created", "pointer_flipped", "failed"], - invoice_created: ["balance_move_sending", "failed", "requires_operator_review"], + invoice_created: [ + "invoice_created", + "balance_move_sending", + "failed", + "requires_operator_review", + ], balance_move_sending: ["balance_move_sent", "failed", "requires_operator_review"], balance_move_sent: ["balance_move_verified", "failed", "requires_operator_review"], balance_move_verified: [ @@ -17,6 +22,7 @@ const transitions: Partial< "requires_operator_review", ], fee_reimbursement_invoice_created: [ + "fee_reimbursement_invoice_created", "fee_reimbursement_sending", "failed", "requires_operator_review", diff --git a/src/app/cash-wallet-cutover/worker.ts b/src/app/cash-wallet-cutover/worker.ts index 50379e31e..f0b860031 100644 --- a/src/app/cash-wallet-cutover/worker.ts +++ b/src/app/cash-wallet-cutover/worker.ts @@ -1,3 +1,5 @@ +import { decodeInvoice } from "@domain/bitcoin/lightning" + import { assertCanTransition } from "./state-machine" import { feeUsdCentsToUsdtMicros, @@ -73,6 +75,23 @@ type CashWalletMigrationProvisioningService = { }): Promise } +const CUTOVER_INVOICE_PAYMENT_SAFETY_WINDOW_MS = 30 * 1000 + +const isInvoicePaymentRequestStale = ({ + paymentRequest, + now, + safetyWindowMs = CUTOVER_INVOICE_PAYMENT_SAFETY_WINDOW_MS, +}: { + paymentRequest: string + now: Date + safetyWindowMs?: number +}): boolean => { + const invoice = decodeInvoice(paymentRequest) + if (invoice instanceof Error) return true + + return invoice.expiresAt.getTime() <= now.getTime() + safetyWindowMs +} + export const startCashWalletMigration = async ({ migration, migrationsRepo, @@ -156,11 +175,17 @@ export const recordCashWalletMigrationBalance = async ({ export const sendCashWalletMigrationBalanceMovePayment = async ({ migration, paymentService, + invoiceService, migrationsRepo, + now = () => new Date(), + invoicePaymentSafetyWindowMs = CUTOVER_INVOICE_PAYMENT_SAFETY_WINDOW_MS, }: { migration: CashWalletMigration paymentService: CashWalletMigrationPaymentService + invoiceService?: CashWalletMigrationNoAmountInvoiceService migrationsRepo: CashWalletMigrationTransitionRepository + now?: () => Date + invoicePaymentSafetyWindowMs?: number }): Promise => { const transition = assertCanTransition(migration.status, "balance_move_sending") if (transition instanceof Error) return transition @@ -177,19 +202,49 @@ export const sendCashWalletMigrationBalanceMovePayment = async ({ ) } + let payableMigration = migration + if ( + invoiceService && + isInvoicePaymentRequestStale({ + paymentRequest: migration.balanceMoveInvoicePaymentRequest, + now: now(), + safetyWindowMs: invoicePaymentSafetyWindowMs, + }) + ) { + const refreshedMigration = await createCashWalletMigrationBalanceMoveInvoice({ + migration, + invoiceService, + migrationsRepo, + }) + if (refreshedMigration instanceof Error) return refreshedMigration + payableMigration = refreshedMigration + } + + if (payableMigration.balanceMoveInvoicePaymentRequest === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "balanceMoveInvoicePaymentRequest is required before balance move payment sending", + ) + } + + if (payableMigration.sourceBalanceUsdCents === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "sourceBalanceUsdCents is required before balance move payment sending", + ) + } + const payment = await paymentService.payInvoice({ - senderWalletId: migration.legacyUsdWalletId, - paymentRequest: migration.balanceMoveInvoicePaymentRequest, - senderAmountUsdCents: migration.sourceBalanceUsdCents, + senderWalletId: payableMigration.legacyUsdWalletId, + paymentRequest: payableMigration.balanceMoveInvoicePaymentRequest, + senderAmountUsdCents: payableMigration.sourceBalanceUsdCents, }) if (payment instanceof Error) return payment return migrationsRepo.transitionMigration({ - id: migration.id, - from: migration.status, + id: payableMigration.id, + from: payableMigration.status, to: "balance_move_sending", - cutoverVersion: migration.cutoverVersion, - runId: migration.runId, + cutoverVersion: payableMigration.cutoverVersion, + runId: payableMigration.runId, patch: { balanceMovePaymentTransactionId: payment.transactionId, }, @@ -367,12 +422,18 @@ export const sendCashWalletMigrationFeeReimbursementPayment = async ({ migration, treasuryWalletId, paymentService, + invoiceService, migrationsRepo, + now = () => new Date(), + invoicePaymentSafetyWindowMs = CUTOVER_INVOICE_PAYMENT_SAFETY_WINDOW_MS, }: { migration: CashWalletMigration treasuryWalletId: WalletId paymentService: CashWalletMigrationPaymentService + invoiceService?: CashWalletMigrationInvoiceService migrationsRepo: CashWalletMigrationTransitionRepository + now?: () => Date + invoicePaymentSafetyWindowMs?: number }): Promise => { const transition = assertCanTransition(migration.status, "fee_reimbursement_sending") if (transition instanceof Error) return transition @@ -383,18 +444,49 @@ export const sendCashWalletMigrationFeeReimbursementPayment = async ({ ) } + let payableMigration = migration + if ( + invoiceService && + isInvoicePaymentRequestStale({ + paymentRequest: migration.feeReimbursementInvoicePaymentRequest, + now: now(), + safetyWindowMs: invoicePaymentSafetyWindowMs, + }) + ) { + if (migration.feeAmountUsdtMicros === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "feeAmountUsdtMicros is required before fee reimbursement invoice refresh", + ) + } + + const refreshedMigration = await createCashWalletMigrationFeeReimbursementInvoice({ + migration, + invoiceService, + migrationsRepo, + feeAmountUsdtMicros: migration.feeAmountUsdtMicros, + }) + if (refreshedMigration instanceof Error) return refreshedMigration + payableMigration = refreshedMigration + } + + if (payableMigration.feeReimbursementInvoicePaymentRequest === undefined) { + return new InvalidCashWalletMigrationTransitionError( + "feeReimbursementInvoicePaymentRequest is required before fee reimbursement sending", + ) + } + const payment = await paymentService.payInvoice({ senderWalletId: treasuryWalletId, - paymentRequest: migration.feeReimbursementInvoicePaymentRequest, + paymentRequest: payableMigration.feeReimbursementInvoicePaymentRequest, }) if (payment instanceof Error) return payment return migrationsRepo.transitionMigration({ - id: migration.id, - from: migration.status, + id: payableMigration.id, + from: payableMigration.status, to: "fee_reimbursement_sending", - cutoverVersion: migration.cutoverVersion, - runId: migration.runId, + cutoverVersion: payableMigration.cutoverVersion, + runId: payableMigration.runId, patch: { feeReimbursementPaymentTransactionId: payment.transactionId, }, diff --git a/src/scripts/cash-wallet-cutover.ts b/src/scripts/cash-wallet-cutover.ts index c417547e7..5e481a074 100644 --- a/src/scripts/cash-wallet-cutover.ts +++ b/src/scripts/cash-wallet-cutover.ts @@ -33,6 +33,8 @@ const args = yargs(hideBin(process.argv)) .option("step-delay-ms", { type: "number", default: 0 }) .option("provision-limit", { type: "number" }) .option("provision-delay-ms", { type: "number", default: 12_500 }) + .option("provision-retry-delay-ms", { type: "number", default: 60_000 }) + .option("max-provision-attempts", { type: "number", default: 5 }) .option("dry-run", { type: "boolean", default: false }) .option("lock-stale-seconds", { type: "number", default: 300 }) .option("configPath", { type: "string", demandOption: true }) @@ -70,6 +72,8 @@ const run = async () => { addWalletIfNonexistent, provisionLimit: args["provision-limit"], provisionDelayMs: args["provision-delay-ms"], + provisionRetryDelayMs: args["provision-retry-delay-ms"], + maxProvisionAttempts: args["max-provision-attempts"], dryRun: args["dry-run"], }) if (result instanceof Error) throw result diff --git a/test/flash/unit/app/cash-wallet-cutover/migration-state-machine.spec.ts b/test/flash/unit/app/cash-wallet-cutover/migration-state-machine.spec.ts index 316fa2443..c6e4c69a3 100644 --- a/test/flash/unit/app/cash-wallet-cutover/migration-state-machine.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/migration-state-machine.spec.ts @@ -37,6 +37,16 @@ describe("cash wallet cutover migration state machine", () => { expect(assertCanTransition("balance_move_verified", "fee_reimbursed")).toBe(true) }) + it("allows invoice refreshes before paying resumable invoices", () => { + expect(assertCanTransition("invoice_created", "invoice_created")).toBe(true) + expect( + assertCanTransition( + "fee_reimbursement_invoice_created", + "fee_reimbursement_invoice_created", + ), + ).toBe(true) + }) + it("resumes from stored checkpoint without repeating completed side effects", () => { expect(nextResumeStatus("invoice_created")).toBe("invoice_created") expect(nextResumeStatus("balance_move_sent")).toBe("balance_move_sent") diff --git a/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts b/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts index 94733f022..9c7610f85 100644 --- a/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/provision-usdt-wallets.spec.ts @@ -151,6 +151,72 @@ describe("provision primary cash wallet USDT wallets", () => { expect(alreadyUsdtAccount.defaultWalletId).toBe("already-usdt-account-usdt") }) + it("backs off and retries rate-limited wallet provisioning before marking it failed", async () => { + const firstAccount = account("first-account" as AccountId, "first-usd" as WalletId) + const secondAccount = account("second-account" as AccountId, "second-usd" as WalletId) + const accounts = [firstAccount, secondAccount] + const walletsByAccountId = new Map([ + [firstAccount.id, [wallet(firstAccount.id, "first-usd" as WalletId, WalletCurrency.Usd)]], + [ + secondAccount.id, + [wallet(secondAccount.id, "second-usd" as WalletId, WalletCurrency.Usd)], + ], + ]) + const firstUsdt = wallet(firstAccount.id, "first-usdt" as WalletId, WalletCurrency.Usdt) + const secondUsdt = wallet( + secondAccount.id, + "second-usdt" as WalletId, + WalletCurrency.Usdt, + ) + const addWalletIfNonexistent = jest + .fn() + .mockResolvedValueOnce(new Error("FetchError: Too Many Requests")) + .mockImplementationOnce(async () => { + walletsByAccountId.set(firstAccount.id, [ + ...(walletsByAccountId.get(firstAccount.id) ?? []), + firstUsdt, + ]) + return firstUsdt + }) + .mockImplementationOnce(async () => { + walletsByAccountId.set(secondAccount.id, [ + ...(walletsByAccountId.get(secondAccount.id) ?? []), + secondUsdt, + ]) + return secondUsdt + }) + const sleep = jest.fn(async () => undefined) + + const result = await provisionPrimaryCashWalletUsdtWallets({ + cutoverVersion: 7, + runId: "run-7", + accountsRepo: { listUnlockedAccounts: () => unlockedAccounts(accounts) }, + walletsRepo: { + listByAccountId: jest.fn( + async (accountId: AccountId) => walletsByAccountId.get(accountId) ?? [], + ), + }, + migrationsRepo: { getConfig: jest.fn(async () => config("pre")) }, + addWalletIfNonexistent, + provisionDelayMs: 1_000, + provisionRetryDelayMs: 30_000, + maxProvisionAttempts: 3, + sleep, + }) + + expect(result).toMatchObject({ + eligible: 2, + provisioned: [ + { accountId: "first-account", walletId: "first-usdt" }, + { accountId: "second-account", walletId: "second-usdt" }, + ], + failed: [], + }) + expect(addWalletIfNonexistent).toHaveBeenCalledTimes(3) + expect(sleep).toHaveBeenCalledWith(30_000) + expect(sleep).toHaveBeenCalledWith(1_000) + }) + it("does not provision after the cutover has started", async () => { const addWalletIfNonexistent = jest.fn() diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts index d02527b9b..b7f9fdd21 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -120,11 +120,13 @@ describe("cash wallet migration runtime services", () => { expect(result).toMatchObject({ paymentRequest: ibexAddInvoiceResponse.invoice.bolt11, }) - expect(Ibex.addInvoice).toHaveBeenCalledWith({ - accountId: "usdt-wallet-id", - memo: "cash-wallet-cutover:run-7:migration-id:balance-move", - expiration: 900, - }) + const args = jest.mocked(Ibex.addInvoice).mock.calls[0][0]! + expect(args.accountId).toBe("usdt-wallet-id") + expect(args.memo).toBe("cash-wallet-cutover:run-7:migration-id:balance-move") + expect(args.expiration).toBe(900) + expect(args.amount).toBeInstanceOf(USDTAmount) + expect((args.amount as USDTAmount).asSmallestUnits()).toBe("0") + expect((args.amount as USDTAmount).toIbex()).toBe(0) }) it("creates amount destination invoices in exact USDT micros through IBEX", async () => { diff --git a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts index fb4699724..8646ee4aa 100644 --- a/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/worker.spec.ts @@ -30,6 +30,9 @@ const migration = (status: CashWalletMigrationStatus): CashWalletMigration => ({ updatedAt: new Date("2026-05-20T00:00:00Z"), }) +const expiredCutoverPaymentRequest = + "lnbc1p4pcau7pp5gaweqhcssvpgcnmqemwhr2vy024yyc9a9lggu2mq9dmxfla939nsdyqvdshx6pdwaskcmr9wskkxat5damx2u36d4skuatpdsknxdfn8geryvesv5cnwepdxuerswfdxsmnxd3dvgenvc3dv9nr2enzxsek2etpxvcr5cnpd3skucm994kk7an9cqzzsxqzpusp5dxuvs8zkzt0tdjkz5ezuea6j49p7yhu43kurz8wcf2xryryp0anq9qxpqysgqnypt73d64vpk74kgdk26s0r7c3yufn2yxpyae3h6zagved5dy2hjek3hxsa3nxqqe5pppqygcrxt6t99tgqc66zet4m99yldkq6muysqvvdhu9" as EncodedPaymentRequest + describe("cash wallet migration worker checkpoints", () => { it("starts a not-started migration with an atomic repository transition", async () => { const startedAt = new Date("2026-05-20T13:00:00Z") @@ -316,6 +319,88 @@ describe("cash wallet migration worker checkpoints", () => { }) }) + it("regenerates an expired balance move invoice before paying it", async () => { + const refreshedInvoice = { + paymentRequest: "lnbc1fresh-balance-move" as EncodedPaymentRequest, + paymentHash: "freshBalanceMoveHash" as PaymentHash, + } as LnInvoice + const refreshedMigration = { + ...migration("invoice_created"), + balanceMoveInvoicePaymentRequest: refreshedInvoice.paymentRequest, + balanceMoveInvoicePaymentHash: refreshedInvoice.paymentHash, + sourceBalanceUsdCents: "1000", + destinationAmountUsdtMicros: "10000000", + } + const migrationsRepo = { + transitionMigration: jest + .fn() + .mockResolvedValueOnce(refreshedMigration) + .mockResolvedValueOnce({ + ...refreshedMigration, + status: "balance_move_sending", + balanceMovePaymentTransactionId: "ibex-tx-id", + }), + } + const invoiceService = { + createNoAmountInvoice: jest.fn(async () => refreshedInvoice), + } + const paymentService = { + payInvoice: jest.fn(async () => ({ + transactionId: "ibex-tx-id" as IbexTransactionId, + })), + } + + const args = { + migration: { + ...migration("invoice_created"), + balanceMoveInvoicePaymentRequest: expiredCutoverPaymentRequest, + balanceMoveInvoicePaymentHash: "expiredBalanceMoveHash" as PaymentHash, + sourceBalanceUsdCents: "1000", + destinationAmountUsdtMicros: "10000000", + }, + paymentService, + invoiceService, + migrationsRepo, + now: () => new Date("2026-05-31T18:04:00Z"), + } + const result = await sendCashWalletMigrationBalanceMovePayment(args) + + expect(result).toMatchObject({ + status: "balance_move_sending", + balanceMovePaymentTransactionId: "ibex-tx-id", + }) + expect(invoiceService.createNoAmountInvoice).toHaveBeenCalledWith({ + recipientWalletId: "usdt-wallet-id", + memo: "cash-wallet-cutover:run-7:migration-id:balance-move", + }) + expect(paymentService.payInvoice).toHaveBeenCalledWith({ + senderWalletId: "legacy-usd-wallet-id", + paymentRequest: "lnbc1fresh-balance-move", + senderAmountUsdCents: "1000", + }) + expect(migrationsRepo.transitionMigration).toHaveBeenNthCalledWith(1, { + id: "migration-id", + from: "invoice_created", + to: "invoice_created", + cutoverVersion: 7, + runId: "run-7", + patch: { + balanceMoveInvoicePaymentRequest: "lnbc1fresh-balance-move", + balanceMoveInvoicePaymentHash: "freshBalanceMoveHash", + }, + }) + expect(migrationsRepo.transitionMigration).toHaveBeenNthCalledWith(2, { + id: "migration-id", + from: "invoice_created", + to: "balance_move_sending", + cutoverVersion: 7, + runId: "run-7", + patch: { + balanceMovePaymentTransactionId: "ibex-tx-id", + }, + }) + }) + it("rejects balance move payment sending when the invoice payment request is missing", async () => { const migrationsRepo = { transitionMigration: jest.fn(), @@ -642,6 +727,91 @@ describe("cash wallet migration worker checkpoints", () => { }) }) + it("regenerates an expired fee reimbursement invoice before paying it", async () => { + const refreshedInvoice = { + paymentRequest: "lnbc1fresh-fee-reimbursement" as EncodedPaymentRequest, + paymentHash: "freshFeeReimbursementHash" as PaymentHash, + } as LnInvoice + const refreshedMigration = { + ...migration("fee_reimbursement_invoice_created"), + feeAmountUsdCents: "8", + feeAmountUsdtMicros: "70001", + feeReimbursementInvoicePaymentRequest: refreshedInvoice.paymentRequest, + feeReimbursementInvoicePaymentHash: refreshedInvoice.paymentHash, + } + const migrationsRepo = { + transitionMigration: jest + .fn() + .mockResolvedValueOnce(refreshedMigration) + .mockResolvedValueOnce({ + ...refreshedMigration, + status: "fee_reimbursement_sending", + feeReimbursementPaymentTransactionId: "fee-ibex-tx-id", + }), + } + const invoiceService = { + createInvoice: jest.fn(async () => refreshedInvoice), + } + const paymentService = { + payInvoice: jest.fn(async () => ({ + transactionId: "fee-ibex-tx-id" as IbexTransactionId, + })), + } + + const args = { + migration: { + ...migration("fee_reimbursement_invoice_created"), + feeAmountUsdCents: "8", + feeAmountUsdtMicros: "70001", + feeReimbursementInvoicePaymentRequest: expiredCutoverPaymentRequest, + feeReimbursementInvoicePaymentHash: "expiredFeeReimbursementHash" as PaymentHash, + }, + treasuryWalletId: "treasury-wallet-id" as WalletId, + paymentService, + invoiceService, + migrationsRepo, + now: () => new Date("2026-05-31T18:04:00Z"), + } + const result = await sendCashWalletMigrationFeeReimbursementPayment(args) + + expect(result).toMatchObject({ + status: "fee_reimbursement_sending", + feeReimbursementPaymentTransactionId: "fee-ibex-tx-id", + }) + expect(invoiceService.createInvoice).toHaveBeenCalledWith({ + recipientWalletId: "usdt-wallet-id", + amount: "80000", + memo: "cash-wallet-cutover:run-7:migration-id:fee-reimbursement", + }) + expect(paymentService.payInvoice).toHaveBeenCalledWith({ + senderWalletId: "treasury-wallet-id", + paymentRequest: "lnbc1fresh-fee-reimbursement", + }) + expect(migrationsRepo.transitionMigration).toHaveBeenNthCalledWith(1, { + id: "migration-id", + from: "fee_reimbursement_invoice_created", + to: "fee_reimbursement_invoice_created", + cutoverVersion: 7, + runId: "run-7", + patch: { + feeAmountUsdCents: "8", + feeAmountUsdtMicros: "70001", + feeReimbursementInvoicePaymentRequest: "lnbc1fresh-fee-reimbursement", + feeReimbursementInvoicePaymentHash: "freshFeeReimbursementHash", + }, + }) + expect(migrationsRepo.transitionMigration).toHaveBeenNthCalledWith(2, { + id: "migration-id", + from: "fee_reimbursement_invoice_created", + to: "fee_reimbursement_sending", + cutoverVersion: 7, + runId: "run-7", + patch: { + feeReimbursementPaymentTransactionId: "fee-ibex-tx-id", + }, + }) + }) + it("rejects fee reimbursement sending when the invoice payment request is missing", async () => { const migrationsRepo = { transitionMigration: jest.fn(), From e84e54da286b2146967992ceef6635043910e5f8 Mon Sep 17 00:00:00 2001 From: Vandana Date: Sun, 31 May 2026 16:33:12 -0700 Subject: [PATCH 4/5] fix(cutover): retry ibex rate limits during migration payments --- .../cash-wallet-cutover/runtime-services.ts | 90 ++++++++++++++++--- .../runtime-services.spec.ts | 27 ++++++ 2 files changed, 103 insertions(+), 14 deletions(-) diff --git a/src/app/cash-wallet-cutover/runtime-services.ts b/src/app/cash-wallet-cutover/runtime-services.ts index 50b2b0f97..4f2766689 100644 --- a/src/app/cash-wallet-cutover/runtime-services.ts +++ b/src/app/cash-wallet-cutover/runtime-services.ts @@ -17,6 +17,10 @@ import { import { destinationShortfallUsdtMicros } from "./amount-conversion" const CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS = 15 * 60 +const CUTOVER_IBEX_RATE_LIMIT_RETRY_DELAY_MS = 60_000 +const CUTOVER_IBEX_RATE_LIMIT_MAX_ATTEMPTS = 5 + +type SleepFn = (delayMs: number) => Promise type RuntimeServiceDependencies = { now?: () => Date @@ -28,6 +32,9 @@ type RuntimeServiceDependencies = { payInvoice?: typeof Ibex.payInvoice accountsRepo?: Pick, "findById"> getTreasuryWalletId?: () => Promise + maxRateLimitAttempts?: number + rateLimitRetryDelayMs?: number + sleep?: SleepFn } const isUsdAmount = (amount: unknown): amount is USDAmount => amount instanceof USDAmount @@ -46,6 +53,40 @@ const ibexInvoiceToDomainInvoice = (response: Awaited + new Promise((resolve) => setTimeout(resolve, delayMs)) + +const errorMessage = (error: Error): string => error.message || String(error) + +const isIbexRateLimitError = (error: Error): boolean => + errorMessage(error).toLowerCase().includes("too many requests") + +const withIbexRateLimitRetry = async ({ + operation, + maxAttempts, + retryDelayMs, + sleepFn, +}: { + operation: () => Promise + maxAttempts: number + retryDelayMs: number + sleepFn: SleepFn +}): Promise => { + for (let attempt = 1; ; attempt += 1) { + const result = await operation() + + if ( + !(result instanceof Error) || + !isIbexRateLimitError(result) || + attempt >= maxAttempts + ) { + return result + } + + await sleepFn(retryDelayMs) + } +} + export const createCashWalletMigrationRuntimeServices = ( deps: RuntimeServiceDependencies = {}, ) => { @@ -56,6 +97,15 @@ export const createCashWalletMigrationRuntimeServices = ( const noAmountInvoiceForRecipient = deps.createNoAmountInvoice ?? Ibex.addInvoice const payInvoice = deps.payInvoice ?? Ibex.payInvoice const accountsRepo = deps.accountsRepo ?? AccountsRepository() + const rateLimitRetry = { + maxAttempts: Math.max( + 1, + deps.maxRateLimitAttempts ?? CUTOVER_IBEX_RATE_LIMIT_MAX_ATTEMPTS, + ), + retryDelayMs: + deps.rateLimitRetryDelayMs ?? CUTOVER_IBEX_RATE_LIMIT_RETRY_DELAY_MS, + sleepFn: deps.sleep ?? sleep, + } return { now: deps.now ?? (() => new Date()), @@ -118,11 +168,15 @@ export const createCashWalletMigrationRuntimeServices = ( const usdtAmount = USDTAmount.smallestUnits(amount) if (usdtAmount instanceof Error) return Promise.resolve(usdtAmount) - return invoiceForRecipient({ - accountId: recipientWalletId as IbexAccountId, - amount: usdtAmount, - memo, - expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, + return withIbexRateLimitRetry({ + ...rateLimitRetry, + operation: () => + invoiceForRecipient({ + accountId: recipientWalletId as IbexAccountId, + amount: usdtAmount, + memo, + expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, + }), }).then(ibexInvoiceToDomainInvoice) }, createNoAmountInvoice: ({ @@ -132,11 +186,15 @@ export const createCashWalletMigrationRuntimeServices = ( recipientWalletId: WalletId memo: string }) => - noAmountInvoiceForRecipient({ - accountId: recipientWalletId, - amount: USDTAmount.ZERO, - memo, - expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, + withIbexRateLimitRetry({ + ...rateLimitRetry, + operation: () => + noAmountInvoiceForRecipient({ + accountId: recipientWalletId, + amount: USDTAmount.ZERO, + memo, + expiration: CUTOVER_IBEX_INVOICE_EXPIRATION_SECONDS as Seconds, + }), }).then(ibexInvoiceToDomainInvoice), }, paymentService: { @@ -155,10 +213,14 @@ export const createCashWalletMigrationRuntimeServices = ( : USDAmount.cents(senderAmountUsdCents) if (send instanceof Error) return send - const payment = await payInvoice({ - accountId: senderWalletId as IbexAccountId, - invoice: paymentRequest as Bolt11, - send, + const payment = await withIbexRateLimitRetry({ + ...rateLimitRetry, + operation: () => + payInvoice({ + accountId: senderWalletId as IbexAccountId, + invoice: paymentRequest as Bolt11, + send, + }), }) if (payment instanceof Error) return payment diff --git a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts index b7f9fdd21..56c40ca9e 100644 --- a/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/runtime-services.spec.ts @@ -175,6 +175,33 @@ describe("cash wallet migration runtime services", () => { expect(paymentArgs.send.asCents()).toBe("1000") }) + it("backs off and retries IBEX rate limits while paying cutover invoices", async () => { + const rateLimit = new Error("FetchError: Too Many Requests") + const sleep = jest.fn(async () => undefined) + const deps = { + payInvoice: jest + .fn() + .mockResolvedValueOnce(rateLimit) + .mockResolvedValueOnce(rateLimit) + .mockResolvedValueOnce({ transaction: { id: "ibex-tx-id" } }), + maxRateLimitAttempts: 3, + rateLimitRetryDelayMs: 1234, + sleep, + } as any + + const services = createCashWalletMigrationRuntimeServices(deps) + + const result = await services.paymentService.payInvoice({ + senderWalletId: "treasury-wallet-id" as WalletId, + paymentRequest: "lnbc1payment", + }) + + expect(result).toEqual({ transactionId: "ibex-tx-id" }) + expect(deps.payInvoice).toHaveBeenCalledTimes(3) + expect(sleep).toHaveBeenCalledTimes(2) + expect(sleep).toHaveBeenCalledWith(1234) + }) + it("returns an error when IBEX payment response has no transaction id", async () => { const services = createCashWalletMigrationRuntimeServices({ payInvoice: jest.fn(async () => ({})), From 6f1b16294af219afde73cb3d013b11050ec79f99 Mon Sep 17 00:00:00 2001 From: Vandana Date: Sun, 31 May 2026 17:00:37 -0700 Subject: [PATCH 5/5] fix(cutover): clarify dashboard run anomalies --- .../cash-wallet-cutover/operator-dashboard.ts | 45 ++++++++--- src/scripts/cash-wallet-cutover-dashboard.ts | 19 ++++- .../operator-dashboard.spec.ts | 79 ++++++++++++++++++- 3 files changed, 128 insertions(+), 15 deletions(-) diff --git a/src/app/cash-wallet-cutover/operator-dashboard.ts b/src/app/cash-wallet-cutover/operator-dashboard.ts index 743319128..63d4f8faa 100644 --- a/src/app/cash-wallet-cutover/operator-dashboard.ts +++ b/src/app/cash-wallet-cutover/operator-dashboard.ts @@ -112,6 +112,7 @@ export type CashWalletCutoverOperatorSnapshot = { usdTotalCents: number usdtTotalMicros: number anomalies: number + watchlistAnomalies: number canStart: boolean blockers: number watchlistAccounts: number @@ -520,6 +521,19 @@ const computeCutoverBalanceAudit = ({ ) if (!destinationWallet) return undefined + if (destinationWallet.balance.status === "loading") { + return { + status: "loading", + sourceUsdCents, + expectedMinimumUsdtMicros, + destinationStartingBalanceUsdtMicros, + currentDestinationBalanceUsdtMicros: 0, + finalDeltaUsdtMicros: 0, + roundingSubsidyUsdtMicros: 0, + shortfallUsdtMicros: 0, + } + } + const currentDestinationBalanceUsdtMicros = destinationWallet.balance.minorUnitsNumber const finalDeltaUsdtMicros = Math.max( @@ -536,12 +550,7 @@ const computeCutoverBalanceAudit = ({ ) return { - status: - destinationWallet.balance.status === "loading" - ? "loading" - : shortfallUsdtMicros > 0 - ? "shortfall" - : "verified", + status: shortfallUsdtMicros > 0 ? "shortfall" : "verified", sourceUsdCents, expectedMinimumUsdtMicros, destinationStartingBalanceUsdtMicros, @@ -570,6 +579,20 @@ export const refreshOperatorAccountCutoverBalanceAudit = < account.usdtWallets[0] if (!destinationWallet) return account + if (destinationWallet.balance.status === "loading") { + return { + ...account, + cutoverBalanceAudit: { + ...audit, + status: "loading", + currentDestinationBalanceUsdtMicros: 0, + finalDeltaUsdtMicros: 0, + roundingSubsidyUsdtMicros: 0, + shortfallUsdtMicros: 0, + }, + } + } + const currentDestinationBalanceUsdtMicros = destinationWallet.balance.minorUnitsNumber const finalDeltaUsdtMicros = Math.max( @@ -590,12 +613,7 @@ export const refreshOperatorAccountCutoverBalanceAudit = < ...account, cutoverBalanceAudit: { ...audit, - status: - destinationWallet.balance.status === "loading" - ? "loading" - : shortfallUsdtMicros > 0 - ? "shortfall" - : "verified", + status: shortfallUsdtMicros > 0 ? "shortfall" : "verified", currentDestinationBalanceUsdtMicros, finalDeltaUsdtMicros, roundingSubsidyUsdtMicros, @@ -915,6 +933,9 @@ export const buildCashWalletCutoverOperatorSnapshot = async ({ usdTotalCents, usdtTotalMicros, anomalies: accounts.filter((account) => account.anomalies.length > 0).length, + watchlistAnomalies: accounts.filter( + (account) => account.watchlisted && account.anomalies.length > 0, + ).length, canStart: blockers === 0, blockers, watchlistAccounts: accounts.filter((account) => account.watchlisted).length, diff --git a/src/scripts/cash-wallet-cutover-dashboard.ts b/src/scripts/cash-wallet-cutover-dashboard.ts index 3db149e0b..a8d3b217f 100644 --- a/src/scripts/cash-wallet-cutover-dashboard.ts +++ b/src/scripts/cash-wallet-cutover-dashboard.ts @@ -405,6 +405,10 @@ const html = ` const auditText = (audit) => { if (!audit) return "-" const cls = audit.status === "verified" ? "ok" : audit.status === "shortfall" ? "bad" : "warn" + if (audit.status === "loading") { + return 'loading' + + '
waiting for balance
' + } return '' + audit.status + '' + '
delta ' + usdt(audit.finalDeltaUsdtMicros) + '
' + '
subsidy ' + usdt(audit.roundingSubsidyUsdtMicros) + '
' + @@ -417,12 +421,22 @@ const html = ` account.usdtWallets.find((w) => w.expected) || account.usdtWallets[0] if (!wallet) return + if (wallet.balance.status === "loading") { + account.cutoverBalanceAudit = Object.assign({}, audit, { + status: "loading", + currentDestinationBalanceUsdtMicros: 0, + finalDeltaUsdtMicros: 0, + roundingSubsidyUsdtMicros: 0, + shortfallUsdtMicros: 0, + }) + return + } const current = wallet.balance.minorUnitsNumber const finalDelta = Math.max(0, current - audit.destinationStartingBalanceUsdtMicros) const shortfall = Math.max(0, audit.expectedMinimumUsdtMicros - finalDelta) const subsidy = Math.max(0, finalDelta - audit.expectedMinimumUsdtMicros) account.cutoverBalanceAudit = Object.assign({}, audit, { - status: wallet.balance.status === "loading" ? "loading" : shortfall > 0 ? "shortfall" : "verified", + status: shortfall > 0 ? "shortfall" : "verified", currentDestinationBalanceUsdtMicros: current, finalDeltaUsdtMicros: finalDelta, roundingSubsidyUsdtMicros: subsidy, @@ -469,7 +483,8 @@ const html = ` metric("Treasury USDT", usdt(s.treasury.usdtTotalMicros)), metric("Customer Total", money(s.reconciliation.customerTotalCents)), metric("System Total", money(s.reconciliation.systemTotalCents)), - metric("Anomalies", s.anomalies, s.anomalies ? "bad" : "ok"), + metric("Run anomalies", s.watchlistAnomalies || 0, s.watchlistAnomalies ? "bad" : "ok"), + metric("Global anomalies", s.anomalies, s.anomalies ? "bad" : "ok"), ].join("") } diff --git a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts index 78ca82170..56121ccfd 100644 --- a/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts +++ b/test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts @@ -115,6 +115,7 @@ describe("cash wallet cutover operator dashboard", () => { usdTotalCents: 123, usdtTotalMicros: 456_000, anomalies: 1, + watchlistAnomalies: 1, canStart: false, blockers: 0, watchlistAccounts: 1, @@ -435,6 +436,74 @@ describe("cash wallet cutover operator dashboard", () => { }) }) + it("does not report an audit shortfall while destination balances are still loading", async () => { + const snapshot = await buildCashWalletCutoverOperatorSnapshot({ + manifestAccounts: [ + { + index: 1, + accountId: "account-1" as AccountId, + expectedUsdWalletId: "usd-1" as WalletId, + expectedUsdtWalletId: "usdt-1" as WalletId, + }, + ], + accountsRepo: { + findById: jest.fn(async () => + account({ + id: "account-1" as AccountId, + defaultWalletId: "usdt-1" as WalletId, + }), + ), + }, + walletsRepo: { + listByAccountId: jest.fn(async () => [ + wallet({ + id: "usd-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usd, + }), + wallet({ + id: "usdt-1" as WalletId, + accountId: "account-1" as AccountId, + currency: WalletCurrency.Usdt, + }), + ]), + }, + migrationsRepo: { + getConfig: jest.fn(async () => ({ + state: "complete" as CashWalletCutoverState, + cutoverVersion: 7, + runId: "run-7", + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + findMigrationByAccountId: jest.fn(async () => ({ + id: "migration-1", + accountId: "account-1" as AccountId, + legacyUsdWalletId: "usd-1" as WalletId, + destinationUsdtWalletId: "usdt-1" as WalletId, + cutoverVersion: 7, + runId: "run-7", + status: "complete" as CashWalletMigrationStatus, + sourceBalanceUsdCents: "10", + destinationAmountUsdtMicros: "100000", + destinationStartingBalanceUsdtMicros: "0", + idempotencyKey: "key", + attempts: 1, + updatedAt: new Date("2026-05-26T20:00:00.000Z"), + })), + }, + getBalanceForWallet: jest.fn(), + balanceMode: "structural", + now: new Date("2026-05-26T20:01:00.000Z"), + }) + + expect(snapshot.accounts[0].cutoverBalanceAudit).toMatchObject({ + status: "loading", + finalDeltaUsdtMicros: 0, + roundingSubsidyUsdtMicros: 0, + shortfallUsdtMicros: 0, + }) + }) + it("includes funder balances in reconciliation without adding migration rows", async () => { const customerUsd = USDAmount.cents(452n) const customerUsdt = USDTAmount.smallestUnits(45_200_00n) @@ -601,6 +670,11 @@ describe("cash wallet cutover operator dashboard", () => { accountId: "global-account" as AccountId, currency: WalletCurrency.Usdt, }), + wallet({ + id: "global-extra-usd" as WalletId, + accountId: "global-account" as AccountId, + currency: WalletCurrency.Usd, + }), ], ], ]) @@ -657,6 +731,8 @@ describe("cash wallet cutover operator dashboard", () => { expect(snapshot.summary.accounts).toBe(2) expect(snapshot.summary.watchlistAccounts).toBe(1) + expect(snapshot.summary.anomalies).toBe(1) + expect(snapshot.summary.watchlistAnomalies).toBe(0) expect(snapshot.accounts.map((row) => row.accountId)).toEqual([ "watchlist-account", "global-account", @@ -666,7 +742,8 @@ describe("cash wallet cutover operator dashboard", () => { expect(snapshot.accounts[1].watchlisted).toBe(false) expect(snapshot.accounts[1].expectedUsdWalletId).toBe("global-usd") expect(snapshot.accounts[1].expectedUsdtWalletId).toBe("global-usdt") - expect(snapshot.accounts[1].anomalies).not.toContain("unexpected_wallet_id") + expect(snapshot.accounts[1].anomalies).toContain("duplicate_usd") + expect(snapshot.accounts[1].anomalies).toContain("unexpected_wallet_id") }) it("retries transient balance read errors before marking a wallet anomalous", async () => {