From b01140aec6f5b50b77d388be77c3694351b66c65 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sun, 19 Jul 2026 15:58:04 +0800 Subject: [PATCH] fix: harden Railway operations and staging checks Separate CI-only secrets from Railway parity, make worker restarts durable, enable the verified ops digest, and support the isolated offline staging profile. Focused tests, live staging checks, and manual GitHub workflow proofs passed; the broad local gate was blocked by another worktree's heavy-test lock. --- .github/workflows/ops-digest.yml | 14 ++++---- docs/deployment-architecture.md | 20 +++++++----- docs/operator-backlog.md | 16 ++++----- docs/staging-setup.md | 2 +- railway.worker.json | 3 +- scripts/check-env-parity.mjs | 38 +++++++++++++++------- scripts/check-indexing.ts | 2 +- scripts/production-readiness.ts | 4 ++- tests/check-indexing-contract.test.ts | 4 +++ tests/production-readiness-offline.test.ts | 7 ++++ tests/railway-config.test.ts | 5 +++ tests/repo-hygiene.test.ts | 32 +++++++++++++++++- 12 files changed, 106 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ops-digest.yml b/.github/workflows/ops-digest.yml index f2aaf92ad..da97469d8 100644 --- a/.github/workflows/ops-digest.yml +++ b/.github/workflows/ops-digest.yml @@ -2,18 +2,16 @@ # hit-rate, answer spend, degraded/truncation rates) into a one-screen summary # and keeps it in a rolling GitHub issue, commenting only when something is off. # -# SHIPPED DISABLED: only workflow_dispatch is active. To enable the daily cadence: -# 1. Add repo variable PROD_HEALTH_URL (e.g. https://) -# 2. Add repo secret HEALTH_DEEP_PROBE_SECRET (same value as the deployment) -# 3. Uncomment the `schedule:` block below. -# Run one workflow_dispatch first and confirm the digest issue looks right. +# The repo variable and matching deployment/GitHub secret are provisioned. +# Keep workflow_dispatch for operator verification and the daily schedule for +# the normal morning digest. name: Ops Digest on: workflow_dispatch: {} - # schedule: - # # 20:00 UTC = 06:00 Australia/Sydney — a morning digest. - # - cron: "0 20 * * *" + schedule: + # 20:00 UTC = 06:00 Australia/Sydney — a morning digest. + - cron: "0 20 * * *" concurrency: group: ops-digest diff --git a/docs/deployment-architecture.md b/docs/deployment-architecture.md index 914ad0f73..71ca441be 100644 --- a/docs/deployment-architecture.md +++ b/docs/deployment-architecture.md @@ -232,7 +232,9 @@ check and watch patterns rather than relying on dashboard defaults. Ship the existing worker as a container (`Dockerfile.worker`: Node 24 + a prebuilt esbuild bundle over production-only `node_modules` + Tesseract + a Python venv with `worker/python/requirements.txt`) and run **one -always-on worker instance** co-located in Railway Singapore (`worker` service). +always-on worker instance** co-located in Railway Singapore (`worker` service), +using Railway's `ALWAYS` restart policy so repeated bootstrap failures cannot +exhaust a finite retry allowance and leave the queue undrained. The `indexing-v3-agent` Edge Function **stays** in its current role as the cron-triggered completion/repair gate — the two are complementary, not alternatives. The worker service selects its Dockerfile via the @@ -301,7 +303,7 @@ Operational rules that follow: - **Worker death costs at most one stale window of latency** for the in-flight job and zero data loss: all artifact writes are idempotent per generation/chunk-key, and completion is gated by the strict completion RPCs - plus the edge agent. Railway's restart-on-failure brings the worker back and + plus the edge agent. Railway's always-restart policy brings the worker back and it reclaims stale jobs automatically. - **Backlog improvement (not in this change):** a heartbeat that refreshes `locked_at` could ride the existing throttled progress updates @@ -342,7 +344,8 @@ Rules: ## 5. Staging environment - **A second, dedicated Supabase project** (same org, ap-southeast-2) — not a - branch of production. Rationale: staging must absorb soak tests, destructive + branch of production. `Clinical KB Staging` was provisioned and migrated on + 2026-07-19. Rationale: staging must absorb soak tests, destructive ingestion experiments, and migration rehearsal without any shared compute, pooling, or the production auth 10-connection cap; per-environment keys fall out naturally. @@ -351,11 +354,12 @@ Rules: synthetic/public corpus. `public/demo-documents/` plus generated samples (`npm run samples`) are sufficient for load-shape realism; do not copy clinical production documents into staging. -- One staging `app` container + one staging `worker` container from the _same_ - images, different env. On Railway this is naturally a **second environment in - the `clinical-kb` project** (e.g. a `staging` environment) or a separate - project, with `RAG_PROVIDER_MODE=auto` and the staging OpenAI key. See - `docs/staging-setup.md` for the turnkey runbook. +- One staging `app` container and **no staging worker**. The active Railway + `Database` project has a `staging` environment pinned to Singapore with + `RAG_PROVIDER_MODE=offline`, isolated Supabase credentials, and no OpenAI key. + This keeps release proofs deterministic and prevents staging ingestion from + draining or mutating production data. See `docs/staging-setup.md` for the + turnkey runbook. - `src/lib/supabase/project.ts` is staging-aware only when both `SUPABASE_STAGING_PROJECT_REF` and `SUPABASE_STAGING_PROJECT_NAME` are set. The declared staging ref must differ from production and every stale project; diff --git a/docs/operator-backlog.md b/docs/operator-backlog.md index ee79a9dfe..d3c8a6259 100644 --- a/docs/operator-backlog.md +++ b/docs/operator-backlog.md @@ -23,7 +23,7 @@ Findings inventory for handover: [audit-handover-2026-07-14.md](audit-handover-2 | Apply drift-codify forward migration (step 1h) | ✅ done | — | Applied and drift/readiness verified 2026-07-13; verify only unless new reviewed drift is found | [database-drift-detection.md](database-drift-detection.md) | | Apply repo-ahead migrations to live (post-2026-07-13) | ⏳ pending | Queue includes `20260717120000`, `20260717170000`, and `20260717171000`; a forward migration to purge private/non-indexed `document_title_words` rows must be merged and reviewed before apply | Zero unsafe title-word rows; `npm run check:drift`; then `eval:retrieval:quality` (36/36) for the corrector | [deploy-corrector-public-titles.md](deploy-corrector-public-titles.md) · [operator-apply-performance-latency-remediation.md](operator-apply-performance-latency-remediation.md) | | Full release gate (bounded OpenAI spend) | ⏳ pending | migrations 1 applied | `npm run verify:release`; `npm run eval:quality -- --rag-only` | [launch-operator-runbook.md §2](launch-operator-runbook.md) | -| Provision staging Supabase project (`Clinical KB Staging`, ap-southeast-2) | ⏳ pending | — | `npm run check:indexing` after `db push` | [staging-setup.md](staging-setup.md) | +| Provision staging Supabase project (`Clinical KB Staging`, ap-southeast-2) | ✅ done | — | Migrated and deployed app-only Railway staging on 2026-07-19; health checks returned ok | [staging-setup.md](staging-setup.md) | | Staging soak + rollback rehearsal on Railway | ⏳ pending | staging provisioned | `scripts/soak-test.ts --confirm-staging` (answer p95 ≤ 25 s) | [launch-operator-runbook.md §4](launch-operator-runbook.md) · [capacity-review.md](capacity-review.md) | | Production deploy to Railway | ✅ done | — | App deployment recorded live 2026-07-14; re-verify with `GET /api/health` and deployment readiness | [deployment-architecture.md](deployment-architecture.md) | @@ -41,13 +41,13 @@ Findings inventory for handover: [audit-handover-2026-07-14.md](audit-handover-2 Each environment gets **separate** service-role + OpenAI keys (per-env blast radius). Placement is a dashboard/CLI action, never committed. -| Secret / config | Status | Where | Notes | -| ------------------------------------------ | ---------- | ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `RAG_QUERY_HASH_SECRET` (prod) | 🔎 verify | Railway runtime secret | GitHub repo secret present since 2026-07-10 (CI green); confirm the SAME value is set in Railway runtime. PIA-2 fail-closed guard requires it at boot (min 16 chars) | -| `HEALTH_DEEP_PROBE_SECRET` (prod + GitHub) | ⏳ pending | Railway runtime + GitHub repo secret | Needed to enable ops-digest. NOT currently a repo secret. Must be set in BOTH Railway (so the app authorizes the deep probe) AND as a GitHub secret with the same value, then uncomment the ops-digest cron | -| `SUPABASE_SERVICE_ROLE_KEY` (per env) | ⏳ pending | Railway runtime secret | accepts the `sb_secret_…` key | -| `OPENAI_API_KEY` (per env) | ⏳ pending | Railway runtime secret | `RAG_PROVIDER_MODE=auto` | -| OpenAI DPA / ZDR execution | ⏳ pending | OpenAI account + legal | app endpoints are ZDR-eligible; execution is operator + legal — see [openai-cross-border-basis.md](openai-cross-border-basis.md) | +| Secret / config | Status | Where | Notes | +| ------------------------------------------ | ---------- | ------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `RAG_QUERY_HASH_SECRET` (prod) | 🔎 verify | Railway runtime secret | GitHub repo secret present since 2026-07-10 (CI green); confirm the SAME value is set in Railway runtime. PIA-2 fail-closed guard requires it at boot (min 16 chars) | +| `HEALTH_DEEP_PROBE_SECRET` (prod + GitHub) | ✅ done | Railway runtime + GitHub repo secret | Matching values were set and the authorized production deep probe returned healthy on 2026-07-19; the ops-digest schedule is enabled. | +| `SUPABASE_SERVICE_ROLE_KEY` (per env) | ⏳ pending | Railway runtime secret | accepts the `sb_secret_…` key | +| `OPENAI_API_KEY` (per env) | ⏳ pending | Railway runtime secret | `RAG_PROVIDER_MODE=auto` | +| OpenAI DPA / ZDR execution | ⏳ pending | OpenAI account + legal | app endpoints are ZDR-eligible; execution is operator + legal — see [openai-cross-border-basis.md](openai-cross-border-basis.md) | ## Disaster-recovery re-creation (does NOT survive a schema restore) diff --git a/docs/staging-setup.md b/docs/staging-setup.md index 14ddcf9ec..223159ca2 100644 --- a/docs/staging-setup.md +++ b/docs/staging-setup.md @@ -46,7 +46,7 @@ those vars are unset. ## B. Staging app host (compute tier) Host: **Railway**, same as production (see `docs/deployment-architecture.md` §2). -Stand staging up as a **second environment** in the `clinical-kb` Railway project, +Stand staging up as a **second environment** in the active `Database` Railway project, with one `app` service pinned to **Southeast Asia (`asia-southeast1-eqsg3a`, Singapore)** — the closest region to the staging Supabase project in Sydney. Do not create a worker for this release profile. diff --git a/railway.worker.json b/railway.worker.json index 9bf05e65e..db76164c5 100644 --- a/railway.worker.json +++ b/railway.worker.json @@ -25,8 +25,7 @@ ] }, "deploy": { - "restartPolicyType": "ON_FAILURE", - "restartPolicyMaxRetries": 10, + "restartPolicyType": "ALWAYS", "multiRegionConfig": { "asia-southeast1-eqsg3a": { "numReplicas": 1 diff --git a/scripts/check-env-parity.mjs b/scripts/check-env-parity.mjs index f5e8a28b3..0e7869324 100644 --- a/scripts/check-env-parity.mjs +++ b/scripts/check-env-parity.mjs @@ -26,7 +26,7 @@ const root = path.join(path.dirname(fileURLToPath(import.meta.url)), ".."); // Vars that MUST be supplied as deployment/CI secrets (never committed). Each is // asserted to exist in the canonical name set below, so this list cannot silently // drift from the schema. -const EXPECTED_SECRETS = [ +export const EXPECTED_GITHUB_SECRETS = [ "SUPABASE_SERVICE_ROLE_KEY", "OPENAI_API_KEY", "RAG_QUERY_HASH_SECRET", @@ -35,6 +35,13 @@ const EXPECTED_SECRETS = [ "E2E_USER_PASSWORD", ]; +export const EXPECTED_RAILWAY_SECRETS = [ + "SUPABASE_SERVICE_ROLE_KEY", + "OPENAI_API_KEY", + "RAG_QUERY_HASH_SECRET", + "HEALTH_DEEP_PROBE_SECRET", +]; + /** Zod schema keys from env.ts: lines shaped like ` NAME: z.…`. */ export function parseEnvSchemaNames(envTsText) { return [...envTsText.matchAll(/^\s*([A-Z][A-Z0-9_]*)\s*:\s*z\./gm)].map((m) => m[1]); @@ -58,16 +65,23 @@ export function computeParity({ canonical, liveNames, expectedSecrets }) { }; } +/** Extract Railway variable names from the CLI's JSON object without exposing values. */ +export function parseRailwayVariableNames(raw) { + const parsed = JSON.parse(raw); + if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") { + throw new Error("Railway variable JSON must be an object"); + } + return Object.keys(parsed); +} + function ghSecretNames() { const raw = execFileSync("gh", ["secret", "list", "--json", "name"], { encoding: "utf8" }); return JSON.parse(raw).map((s) => s.name); } function railwayVarNames() { - // `railway variables` prints a table; --json/-k vary by CLI version, so parse - // UPPER_SNAKE tokens defensively. Names only. - const raw = execFileSync("railway", ["variables"], { encoding: "utf8" }); - return [...new Set([...raw.matchAll(/\b([A-Z][A-Z0-9_]{2,})\b/g)].map((m) => m[1]))]; + const raw = execFileSync("railway", ["variable", "list", "--json"], { encoding: "utf8" }); + return parseRailwayVariableNames(raw); } function main() { @@ -81,7 +95,8 @@ function main() { const problems = []; // Self-consistency: every expected secret must be a name the app/CI actually knows. - const unknownExpected = EXPECTED_SECRETS.filter((name) => !canonical.has(name)); + const expectedSecrets = new Set([...EXPECTED_GITHUB_SECRETS, ...EXPECTED_RAILWAY_SECRETS]); + const unknownExpected = [...expectedSecrets].filter((name) => !canonical.has(name)); if (unknownExpected.length > 0) { problems.push( `Expected-secret names not found in env.ts/check-ci-env (typo or drift): ${unknownExpected.join(", ")}`, @@ -89,11 +104,12 @@ function main() { } console.log(`Known env names: ${canonical.size} (env.ts schema + check-ci-env).`); - console.log(`Expected secrets: ${EXPECTED_SECRETS.join(", ")}`); + console.log(`Expected GitHub secrets: ${EXPECTED_GITHUB_SECRETS.join(", ")}`); + console.log(`Expected Railway secrets: ${EXPECTED_RAILWAY_SECRETS.join(", ")}`); - for (const [flag, enabled, label, getter] of [ - ["--gh", useGh, "GitHub secrets", ghSecretNames], - ["--railway", useRailway, "Railway variables", railwayVarNames], + for (const [flag, enabled, label, getter, sourceExpectedSecrets] of [ + ["--gh", useGh, "GitHub secrets", ghSecretNames, EXPECTED_GITHUB_SECRETS], + ["--railway", useRailway, "Railway variables", railwayVarNames, EXPECTED_RAILWAY_SECRETS], ]) { if (!enabled) { console.log(`(${label}: skipped — pass ${flag} to check; names only, no values)`); @@ -109,7 +125,7 @@ function main() { const { missingSecrets, unknownLive } = computeParity({ canonical: [...canonical], liveNames, - expectedSecrets: EXPECTED_SECRETS, + expectedSecrets: sourceExpectedSecrets, }); console.log(`\n${label}: ${liveNames.length} names.`); if (missingSecrets.length > 0) problems.push(`${label}: missing expected secret(s): ${missingSecrets.join(", ")}`); diff --git a/scripts/check-indexing.ts b/scripts/check-indexing.ts index 9fec3f42d..98a9cedf8 100644 --- a/scripts/check-indexing.ts +++ b/scripts/check-indexing.ts @@ -361,7 +361,7 @@ async function main() { ]); requireServerEnv(); - requireOpenAIEnv(); + if (env.RAG_PROVIDER_MODE !== "offline") requireOpenAIEnv(); // IDX-C2 fail-fast: verify the configured embedding dimension matches the schema's // vector(N) columns before any indexing/DB work, so a model/dimension misconfiguration diff --git a/scripts/production-readiness.ts b/scripts/production-readiness.ts index 85e84be85..bdd446ce4 100644 --- a/scripts/production-readiness.ts +++ b/scripts/production-readiness.ts @@ -277,9 +277,11 @@ async function main() { NEXT_PUBLIC_SUPABASE_URL: process.env.NEXT_PUBLIC_SUPABASE_URL, SUPABASE_PROJECT_REF: process.env.SUPABASE_PROJECT_REF, SUPABASE_PROJECT_NAME: process.env.SUPABASE_PROJECT_NAME, + SUPABASE_STAGING_PROJECT_REF: process.env.SUPABASE_STAGING_PROJECT_REF, + SUPABASE_STAGING_PROJECT_NAME: process.env.SUPABASE_STAGING_PROJECT_NAME, }); if (supabaseCheck.status === "ready") { - result.passes.push("Supabase project config points to Clinical KB Database."); + result.passes.push(`Supabase project config points to ${supabaseCheck.expected.name}.`); } else if (supabaseCheck.status === "warning") { if (supabaseCheck.warnings.length) { result.warnings.push(...supabaseCheck.warnings); diff --git a/tests/check-indexing-contract.test.ts b/tests/check-indexing-contract.test.ts index 76d9caae2..f00d45db1 100644 --- a/tests/check-indexing-contract.test.ts +++ b/tests/check-indexing-contract.test.ts @@ -10,6 +10,10 @@ const source = readFileSync(new URL("../scripts/check-indexing.ts", import.meta. const smartBackfillSource = readFileSync(new URL("../scripts/backfill-smart-index.ts", import.meta.url), "utf8"); describe("indexing health scan", () => { + it("does not require an OpenAI key for the staging offline profile", () => { + expect(source).toContain('if (env.RAG_PROVIDER_MODE !== "offline") requireOpenAIEnv();'); + }); + it("uses bounded document batches large enough for the production corpus", () => { const batchSize = Number(source.match(/const documentIdBatchSize = (\d+);/)?.[1]); expect(batchSize).toBeGreaterThanOrEqual(50); diff --git a/tests/production-readiness-offline.test.ts b/tests/production-readiness-offline.test.ts index a268a350e..adb67ef8f 100644 --- a/tests/production-readiness-offline.test.ts +++ b/tests/production-readiness-offline.test.ts @@ -1,8 +1,15 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { openAIReadinessPolicy } from "../scripts/production-readiness"; describe("production readiness provider policy", () => { + it("passes the explicit staging declaration to the shared project guard", () => { + const source = readFileSync(new URL("../scripts/production-readiness.ts", import.meta.url), "utf8"); + expect(source).toContain("SUPABASE_STAGING_PROJECT_REF: process.env.SUPABASE_STAGING_PROJECT_REF"); + expect(source).toContain("SUPABASE_STAGING_PROJECT_NAME: process.env.SUPABASE_STAGING_PROJECT_NAME"); + }); + it("requires an OpenAI key for auto and openai modes", () => { expect(openAIReadinessPolicy("auto")).toEqual({ required: true, ready: false }); expect(openAIReadinessPolicy("openai")).toEqual({ required: true, ready: false }); diff --git a/tests/railway-config.test.ts b/tests/railway-config.test.ts index db645b29c..d39bffcaf 100644 --- a/tests/railway-config.test.ts +++ b/tests/railway-config.test.ts @@ -46,6 +46,11 @@ describe("Railway config as code", () => { }); }); + it("keeps the queue-draining worker alive after repeated failures", () => { + expect(worker.deploy).toMatchObject({ restartPolicyType: "ALWAYS" }); + expect(worker.deploy).not.toHaveProperty("restartPolicyMaxRetries"); + }); + it.each([ "Dockerfile", ".dockerignore", diff --git a/tests/repo-hygiene.test.ts b/tests/repo-hygiene.test.ts index 84ac6cff7..6e23dda56 100644 --- a/tests/repo-hygiene.test.ts +++ b/tests/repo-hygiene.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from "vitest"; -import { computeParity, parseCiEnvNames, parseEnvSchemaNames } from "../scripts/check-env-parity.mjs"; +import { + computeParity, + EXPECTED_GITHUB_SECRETS, + EXPECTED_RAILWAY_SECRETS, + parseCiEnvNames, + parseEnvSchemaNames, + parseRailwayVariableNames, +} from "../scripts/check-env-parity.mjs"; import { hasCompletedCleanupReview, parseLedgerBranches } from "../scripts/sweep-branch-ledger.mjs"; describe("check-env-parity name parsing", () => { @@ -34,6 +41,29 @@ describe("check-env-parity name parsing", () => { expect(parity.missingSecrets).toEqual(["SUPABASE_SERVICE_ROLE_KEY"]); expect(parity.unknownLive).toEqual(["LEFTOVER_OLD_KEY"]); }); + + it("keeps CI-only E2E credentials out of Railway expectations", () => { + expect(EXPECTED_GITHUB_SECRETS).toEqual( + expect.arrayContaining(["E2E_USER_EMAIL", "E2E_USER_PASSWORD", "HEALTH_DEEP_PROBE_SECRET"]), + ); + expect(EXPECTED_RAILWAY_SECRETS).toEqual( + expect.arrayContaining([ + "SUPABASE_SERVICE_ROLE_KEY", + "OPENAI_API_KEY", + "RAG_QUERY_HASH_SECRET", + "HEALTH_DEEP_PROBE_SECRET", + ]), + ); + expect(EXPECTED_RAILWAY_SECRETS).not.toEqual(expect.arrayContaining(["E2E_USER_EMAIL", "E2E_USER_PASSWORD"])); + }); + + it("extracts Railway JSON names without treating values as names", () => { + expect( + parseRailwayVariableNames( + JSON.stringify({ OPENAI_API_KEY: "secret-with-UPPERCASE-fragments", RAG_PROVIDER_MODE: "offline" }), + ), + ).toEqual(["OPENAI_API_KEY", "RAG_PROVIDER_MODE"]); + }); }); describe("sweep-branch-ledger parsing", () => {