fix(cutover): harden operator cutover run

.env

@@ -123,8 +123,8 @@ export ERPNEXT_JWT_SECRET="not-so-secret"
 
 COMPOSE_FILE=docker-compose.yml:docker-compose.override.yml:docker-compose.local.yml
 
-# Ibex API (used by docker compose; app reads from yaml config)
-export IBEX_URL="https://api-sandbox.poweredbyibex.io"
-export IBEX_EMAIL=""
-export IBEX_PASSWORD=""
+# Ibex API — app reads from yaml config, not env vars
+# Get sandbox credentials from your team lead
+# export IBEX_CLIENT_ID=""
+# export IBEX_CLIENT_SECRET=""
 

DEV.md

@@ -38,30 +38,11 @@ If you prefer to set things up yourself, or if the setup script fails:
 
 ### 1. Environment Variables
 
-The project loads environment variables from `.env` (committed) and `.env.local` (git-ignored, for secrets).
-
-Create `.env.local` with your Ibex sandbox credentials:
-
-```bash
-echo "export IBEX_EMAIL='your-ibex-email'" >> .env.local
-echo "export IBEX_PASSWORD='your-ibex-password'" >> .env.local
-```
-
-If you use direnv, allow it:
-
-```bash
-direnv allow
-```
-
-If not using direnv, source the env files manually before running commands:
-
-```bash
-source .env && source .env.local
-```
+Flash uses YAML config files. Ibex OAuth2 credentials go in local config overrides, not env vars.
 
 ### 2. App Config Overrides
 
-Flash uses YAML config files. The base config is at `dev/config/base-config.yaml`. Secrets and local overrides go in `$CONFIG_PATH/dev-overrides.yaml` (default: `~/.config/flash/dev-overrides.yaml`).
+The base config is at `dev/config/base-config.yaml`. Secrets and local overrides go in `$CONFIG_PATH/dev-overrides.yaml` (default: `~/.config/flash/dev-overrides.yaml`).
 
 **Option A — Run the interactive script:**
 
@@ -74,8 +55,9 @@ Flash uses YAML config files. The base config is at `dev/config/base-config.yaml
 ```yaml
 # ~/.config/flash/dev-overrides.yaml
 ibex:
-  email: your-ibex-email
-  password: your-ibex-password
+  clientId: your-sandbox-client-id
+  clientSecret: your-sandbox-client-secret
+  environment: sandbox
 ```
 
 Additional overrides you might need:

dev/apollo-federation/supergraph.graphql

@@ -343,6 +343,7 @@ type BridgeWithdrawal
   amount: String!
   createdAt: String!
   currency: String!
+  failureReason: String
   id: ID!
   status: String!
 }

dev/bruno/Flash GraphQL API/environments/local.bru

@@ -1,13 +1,15 @@
 vars {
   flashGraphqlUrl: http://localhost:4002/graphql
   admin_url: http://localhost:4001/graphql
-  admin_token: eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VySWQiOiJhZG1pbiIsInJvbGVzIjpbIkFjY291bnRzIE1hbmFnZXIiXX0.UOmQR2K6RdS1FVvQbjvSQfoQ-VsTC6Y7x2YAXZImdsA
   currency: BTC
-  phone: +16505554322
+  phone: +16505554320
   code: 000000
-  token:
   walletId:
   walletIdUsd: c593736e-5a58-42e4-93fa-dc895856c1f1
   userEmail: mauriente@gmail.com
   userFullName: maurientes
 }
+vars:secret [
+  admin_token,
+  token
+]

dev/config/set-overrides.sh

@@ -7,8 +7,9 @@ mkdir -p "$(dirname "$OUTPUT_FILE")"
 
 # Define YAML paths and their descriptions directly in the script
 declare -a yaml_paths=(
-  "ibex.email, Email address to Ibex Account"
-  "ibex.password, Password to Ibex Account"
+  "ibex.clientId, OAuth2 client ID for the Ibex account"
+  "ibex.clientSecret, OAuth2 client secret for the Ibex account"
+  "ibex.environment, Ibex environment: sandbox or production"
   "ibex.webhook.uri, The URI where Ibex will send payment events"
   "sendgrid.apiKey, API key to SendGrid email service (from Twilio)"
   "cashout.email.to, Recipient email address for cashout notifications"
@@ -57,4 +58,4 @@ for entry in "${yaml_paths[@]}"; do
   write_yaml "$path" $value
 done
 
-echo "YAML file has been written to $OUTPUT_FILE."
+echo "YAML file has been written to $OUTPUT_FILE."

dev/setup.sh

@@ -73,24 +73,27 @@ echo ""
 # ── 5. Configure Ibex credentials ────────────────────
 echo "Checking Ibex credentials..."
 
-if [ -f .env.local ] && grep -q "IBEX_PASSWORD" .env.local 2>/dev/null; then
+if [ -f .env.local ] && grep -q "IBEX_CLIENT_SECRET" .env.local 2>/dev/null; then
   info "Ibex credentials found in .env.local"
 else
   echo ""
-  echo "Flash requires Ibex sandbox credentials to connect to the payment backend."
+  echo "Flash requires Ibex OAuth2 sandbox credentials to connect to the payment backend."
   echo "If you don't have credentials, ask your team lead."
   echo ""
-  read -rp "Ibex email (or press Enter to skip): " IBEX_EMAIL
-  if [ -n "$IBEX_EMAIL" ]; then
-    read -rsp "Ibex password: " IBEX_PASSWORD
+  read -rp "Ibex client ID (or press Enter to skip): " IBEX_CLIENT_ID
+  if [ -n "$IBEX_CLIENT_ID" ]; then
+    read -rsp "Ibex client secret: " IBEX_CLIENT_SECRET
     echo ""
+    read -rp "Ibex environment [sandbox]: " IBEX_ENVIRONMENT
+    IBEX_ENVIRONMENT="${IBEX_ENVIRONMENT:-sandbox}"
     cat > .env.local << EOF
-export IBEX_EMAIL='${IBEX_EMAIL}'
-export IBEX_PASSWORD='${IBEX_PASSWORD}'
+export IBEX_CLIENT_ID='${IBEX_CLIENT_ID}'
+export IBEX_CLIENT_SECRET='${IBEX_CLIENT_SECRET}'
+export IBEX_ENVIRONMENT='${IBEX_ENVIRONMENT}'
 EOF
     info "Credentials saved to .env.local (git-ignored)"
   else
-    warn "Skipped — you'll need to create .env.local with IBEX_EMAIL and IBEX_PASSWORD before starting"
+    warn "Skipped — you'll need to create .env.local with IBEX_CLIENT_ID and IBEX_CLIENT_SECRET before starting"
   fi
 fi
 
@@ -109,11 +112,12 @@ else
   if [ -f .env.local ]; then
     source .env.local 2>/dev/null || true
   fi
-  if [ -n "${IBEX_EMAIL:-}" ] && [ -n "${IBEX_PASSWORD:-}" ]; then
+  if [ -n "${IBEX_CLIENT_ID:-}" ] && [ -n "${IBEX_CLIENT_SECRET:-}" ]; then
     cat > "$OVERRIDES" << EOF
 ibex:
-  email: ${IBEX_EMAIL}
-  password: ${IBEX_PASSWORD}
+  clientId: ${IBEX_CLIENT_ID}
+  clientSecret: ${IBEX_CLIENT_SECRET}
+  environment: ${IBEX_ENVIRONMENT:-sandbox}
 EOF
     info "Generated $OVERRIDES with Ibex credentials"
   else

docker-compose.yml

@@ -79,9 +79,8 @@ services:
       - REDIS_TYPE=standalone
       - REDIS_0_DNS=redis
       - REDIS_0_PORT=6378
-      - IBEX_URL=${IBEX_URL}
-      - IBEX_EMAIL=${IBEX_EMAIL}
-      - IBEX_PASSWORD=${IBEX_PASSWORD}
+      - IBEX_CLIENT_ID=${IBEX_CLIENT_ID:-}
+      - IBEX_CLIENT_SECRET=${IBEX_CLIENT_SECRET:-}
   price-history:
     image: docker.io/lnflash/price-history:edge
     # image: us.gcr.io/galoy-org/price-history:edge

docs/plans/2026-05-26-local-cutover-operator-dashboard.md

@@ -0,0 +1,88 @@
+# Local Cash Wallet Cutover Operator Dashboard Plan
+
+## Goal
+
+Build a local-only dashboard at `http://localhost:3450` that lets an operator monitor the 60 cutover test accounts and their cash wallets through each cutover state. The dashboard must use raw backend repositories and wallet balances, not the GraphQL presentation layer, because public wallet queries intentionally hide either USD or USDT depending on client capability and cutover state.
+
+## Constraints
+
+- Read-only: the dashboard must not mutate accounts, wallets, balances, migrations, or cutover config.
+- Local-only: bind to localhost and serve a static browser UI plus a JSON snapshot endpoint.
+- Source of truth:
+  - account manifests from `/tmp/eng345usd-20260526115410-local-backend-accounts.json` and `/tmp/eng345usdonly-20260526195758-accounts.json`
+  - raw Mongo repositories for accounts, wallets, and cash-wallet-cutover state
+  - `Wallets.getBalanceForWallet` for live balances
+- Expected population:
+  - 60 accounts
+  - 110 current wallets before `provision-usdt-wallets`
+  - 120 target wallets after every USD-only account receives USDT
+- No production API behavior should change.
+
+## Design
+
+1. Add a pure dashboard snapshot builder under `src/app/cash-wallet-cutover/operator-dashboard.ts`.
+   - Load account IDs from manifest records.
+   - Fetch each account by id and all raw wallets by account id.
+   - Identify checking USD and checking USDT wallets directly from raw wallet records.
+   - Fetch live balances for each cash wallet.
+   - Fetch cutover config and per-account migration records when config has `runId`.
+   - Derive summary totals and account-level anomaly badges.
+
+2. Add focused unit tests under `test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts`.
+   - Verify wallet grouping and current/target wallet counts.
+   - Verify missing-USDT detection for USD-only accounts.
+   - Verify funded USD-only count from USD cent balances.
+   - Verify migration status counts and anomaly flags.
+
+3. Add a thin local HTTP script under `src/scripts/cash-wallet-cutover-dashboard.ts`.
+   - Accept `--port`, `--configPath`, optional `--run-id`, optional `--cutover-version`, optional `--expected-accounts`, and repeated `--manifest` arguments.
+   - Default port: `3450`.
+   - Bind explicitly to `127.0.0.1`.
+   - Default manifests:
+     - `/tmp/eng345usd-20260526115410-local-backend-accounts.json`
+     - `/tmp/eng345usdonly-20260526195758-accounts.json`
+   - Routes:
+     - `GET /` static dashboard HTML/CSS/JS
+     - `GET /api/snapshot` live JSON snapshot
+   - Poll snapshot every 10 seconds from the browser, with a manual refresh button.
+   - Cache server-side snapshots for a short TTL so browser refreshes do not hammer IBEX.
+
+4. UI content.
+   - Compact operational layout.
+   - Summary strip for cutover state, run id/version, accounts, wallets current/target, missing USDT, funded USD-only, USD total, USDT total, and anomalies.
+   - Filters for anomalies, funded only, missing USDT, nonzero USD, nonzero USDT, and migration status.
+   - Per-account table with phone, account id, default wallet, USD wallet/balance, USDT wallet/balance, migration status, and anomaly badges.
+   - Color coding:
+     - green expected
+     - yellow pending/missing-but-expected
+     - red broken or dangerous anomalies
+
+5. Verification.
+   - Run the new unit test first and confirm it fails before implementation.
+   - Implement the snapshot builder and dashboard script.
+   - Run the focused unit test.
+   - Run TypeScript check for touched files through the repo build/test path where practical.
+   - Start the dashboard on `localhost:3450` and verify:
+     - `GET /` returns HTML
+     - `GET /api/snapshot` returns JSON
+     - dashboard process is listening on port `3450`
+
+## Risks
+
+- `Wallets.getBalanceForWallet` returns currency-specific amount shapes; the snapshot builder must normalize cautiously and preserve raw balance display for unknown shapes.
+- Account manifest shape may differ between the 50-account and 10-account batches. The loader should accept common `accountId`, `account.id`, `id`, `phone`, and `username` fields and fail with clear errors if no account id can be found.
+- Prepared migration records may exist before cutover config has `runId`. The dashboard must accept explicit `--run-id` and `--cutover-version` and use them for migration lookup when provided.
+- The manifest loader must support the actual top-level `accounts` and `created` arrays, reject duplicate account IDs, and by default validate that 60 accounts were loaded.
+- Balance reads can be expensive across 110-120 wallets. The local server must cache snapshots with a short TTL, capture per-wallet balance errors, and avoid making every browser poll trigger a full IBEX balance sweep.
+- Running through `ts-node` may need `transpile-only` and `tsconfig-paths/register`, matching the earlier local script behavior.
+- Large account lists should remain cheap: 60 accounts and 120 wallets is small, so simple sequential fetches are acceptable for operator clarity.
+
+## Dual-Model Review Notes
+
+- Reviewer 1 required explicit migration lookup arguments so PRE/prepared migration records remain visible. Plan updated.
+- Reviewer 1 required explicit currency on balance reads. Implementation will always pass `wallet.currency`.
+- Reviewer 1 required support for both manifest shapes and default count validation. Plan updated.
+- Reviewer 1 required loopback-only binding. Plan updated.
+- Reviewer 1 recommended keeping the dashboard out of GraphQL/production HTTP routes. The module will only be consumed by the local script and unit tests.
+- Reviewer 2 required server-side snapshot caching and a slower poll interval to avoid roughly 55-60 IBEX calls/sec. Plan updated.
+- Reviewer 2 required dependency injection for the snapshot builder. The builder will take manifests, repos, cutover repo, and `getBalanceForWallet`.

docs/plans/2026-05-28-cutover-dashboard-lazy-balances.md

@@ -0,0 +1,77 @@
+# Cash Wallet Cutover Dashboard Lazy Balances Implementation Plan
+
+> Implementation note: keep this local dashboard read-only and preserve the existing IBEX balance throttle.
+
+**Goal:** Make the local Cash Wallet Cutover Dashboard render readiness and account structure immediately while IBEX wallet balances hydrate lazily in the background.
+
+**Architecture:** Split dashboard data into a fast structural snapshot and a throttled balance refresh path. The structural snapshot reads Mongo, migrations, and preflight state, but does not call IBEX. A local in-memory balance cache and single-worker queue refresh wallet balances with the existing throttle and expose cached values through a lazy `/api/balances` endpoint.
+
+**Tech Stack:** TypeScript, Express, existing Flash repositories, Jest unit tests, vanilla browser JavaScript.
+
+---
+
+## Task 1: Structural Snapshot Mode
+
+**Files:**
+- Modify: `src/app/cash-wallet-cutover/operator-dashboard.ts`
+- Test: `test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts`
+
+**Steps:**
+1. Add a failing unit test proving `buildCashWalletCutoverOperatorSnapshot` can produce rows without calling `getBalanceForWallet` when balance mode is disabled.
+2. Add a placeholder balance formatter that returns `display: "loading"`, zero minor units, and a `status` field for balance hydration.
+3. Thread a `balanceMode` option through the snapshot builder with default live behavior preserved for existing callers.
+4. Verify existing live-balance tests still pass.
+
+## Task 2: Balance Cache And Queue
+
+**Files:**
+- Modify: `src/scripts/cash-wallet-cutover-dashboard.ts`
+
+**Steps:**
+1. Add a focused unit-testable helper only if it can stay small; otherwise keep the cache local to the script.
+2. Add an in-memory `Map<WalletId, CachedBalance>` and FIFO queue with de-duping.
+3. Keep the existing one-wallet-at-a-time throttle and retry behavior inside the queue worker.
+4. Add cache statuses for the first pass: `loading`, `fresh`, and `error`.
+
+## Task 3: Lazy Balance Endpoints
+
+**Files:**
+- Modify: `src/scripts/cash-wallet-cutover-dashboard.ts`
+
+**Steps:**
+1. Change `/api/snapshot` to build structural snapshots only.
+2. Add `GET /api/balances?walletIds=...&refresh=0|1`, returning cached balance payloads immediately and enqueueing requested wallet IDs.
+3. Add `GET /api/balance-status` for queue length, refreshed count, loading count, and last sweep timestamp.
+4. Keep `?refresh=1` on `/api/snapshot` as structural refresh only, not a full IBEX balance sweep.
+
+## Task 4: Browser Lazy Hydration
+
+**Files:**
+- Modify: `src/scripts/cash-wallet-cutover-dashboard.ts`
+
+**Steps:**
+1. Render the structural snapshot immediately.
+2. Collect wallet IDs from the structural snapshot and hand them to `/api/balances` without blocking first render.
+3. Poll `/api/balances` and update the row objects in memory as balances arrive.
+4. Show status text such as `Balances 24/192 refreshed` instead of blocking `Loading...`.
+5. Ensure filters continue to work while balances are loading.
+
+## Task 5: Verification
+
+**Commands:**
+1. Run focused unit tests:
+   `PATH=/Users/dread/.nvm/versions/node/v20.20.0/bin:$PATH TEST=test/flash/unit/app/cash-wallet-cutover/operator-dashboard.spec.ts yarn test:unit`
+2. Restart local dashboard:
+   `tmux kill-session -t cutover-dashboard` then start the existing dashboard command.
+3. Verify:
+   - `GET /` returns HTML.
+   - `GET /api/snapshot?refresh=1` returns quickly and reports `watchlistAccounts: 60`.
+   - `GET /api/balances` returns immediately with cached/loading payloads.
+   - `GET /api/balance-status` shows queue progress.
+
+## Constraints
+
+- Do not increase IBEX request rate.
+- Do not mutate accounts, wallets, migrations, or cutover config.
+- Keep dashboard local-only on `127.0.0.1`.
+- Avoid broad refactors and generated-file churn.

src/app/cash-wallet-cutover/handlers.ts

@@ -114,6 +114,8 @@ export const createCashWalletMigrationStepHandlers = ({
       migration,
       migrationsRepo,
       paymentService: services.paymentService,
+      invoiceService: services.invoiceService,
+      now: services.now,
     }),
   balance_move_sending: (migration) =>
     markCashWalletMigrationBalanceMoveSent({ migration, migrationsRepo }),
@@ -147,6 +149,8 @@ export const createCashWalletMigrationStepHandlers = ({
       migration,
       migrationsRepo,
       paymentService: services.paymentService,
+      invoiceService: services.invoiceService,
+      now: services.now,
       treasuryWalletId,
     })
   },

src/app/cash-wallet-cutover/index.ts

@@ -21,3 +21,4 @@ export * from "./runtime-services"
 export * from "./orchestrator"
 export * from "./lifecycle"
 export * from "./preview"
+export * from "./provision-usdt-wallets"