diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..77dc8a4 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,37 @@ +name: e2e + +# Gated end-to-end suite against a real GitHub test org. Never runs on PRs — +# nightly + on demand only, so it can't block contributions and only runs where +# the WARDEN_E2E_* secrets exist. The suite self-skips if they're unset. +on: + schedule: + - cron: '0 7 * * *' # 07:00 UTC nightly + workflow_dispatch: + inputs: + apply: + description: 'Also run the mutating Phase 2 (teardown-guarded)' + type: boolean + default: false + +permissions: + contents: read + +jobs: + e2e: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: '24' + cache: npm + - run: npm ci + - name: e2e + env: + WARDEN_E2E_APP_ID: ${{ secrets.WARDEN_E2E_APP_ID }} + WARDEN_E2E_INSTALLATION_ID: ${{ secrets.WARDEN_E2E_INSTALLATION_ID }} + WARDEN_E2E_PRIVATE_KEY: ${{ secrets.WARDEN_E2E_PRIVATE_KEY }} + WARDEN_E2E_ORG: ${{ secrets.WARDEN_E2E_ORG }} + WARDEN_E2E_APPLY: ${{ inputs.apply && '1' || '' }} + run: npm run test:e2e diff --git a/README.md b/README.md index da1c09d..d6c5cbf 100644 --- a/README.md +++ b/README.md @@ -249,6 +249,29 @@ way the chant lexicons publish. `just release [patch|minor|major]` bumps, tags `vX.Y.Z`, and pushes; `publish.yml` then publishes with `id-token: write` + `--provenance`. +## End-to-end tests + +Unit tests (`npm test`) are fully mocked. A separate **gated** e2e suite exercises +every cycle against a **real GitHub org** via a real App installation — it's the +only thing that validates the live API contract (especially the App-only token +cycles). It's excluded from `npm test` and from PR CI. + +```bash +WARDEN_E2E_APP_ID=… WARDEN_E2E_INSTALLATION_ID=… \ +WARDEN_E2E_PRIVATE_KEY="$(cat key.pem)" WARDEN_E2E_ORG=my-test-org \ +npm run test:e2e +``` + +- **Phase 1 (always):** for each cycle, runs `fetchLive` + `diff` against the org + and asserts every HTTP call was a `GET` (fetchLive never mutates) and the + pipeline composes — this catches API drift. The suite **self-skips** when the + `WARDEN_E2E_*` vars are unset. +- **Phase 2 (opt-in, `WARDEN_E2E_APPLY=1`):** one teardown-guarded mutation + (create + delete a repo Actions variable) to prove the write path. + +CI runs it nightly + on demand via `.github/workflows/e2e.yml` using +`WARDEN_E2E_*` repo secrets (never on PRs). + ## Architecture The provider-agnostic reconcile core (change-set model, generic collection diff, diff --git a/e2e/warden.e2e.test.ts b/e2e/warden.e2e.test.ts new file mode 100644 index 0000000..08ab093 --- /dev/null +++ b/e2e/warden.e2e.test.ts @@ -0,0 +1,184 @@ +/** + * End-to-end harness — exercises warden's cycles against a REAL GitHub org via + * a real GitHub App installation. Gated and excluded from the default test run + * (`vitest.config.ts` only globs `src/**`); run with `npm run test:e2e`. + * + * The whole suite SKIPS unless these env vars are set, so default CI and + * contributors without a test org are unaffected: + * + * WARDEN_E2E_APP_ID GitHub App id + * WARDEN_E2E_INSTALLATION_ID installation id on the test org + * WARDEN_E2E_PRIVATE_KEY App private key PEM + * WARDEN_E2E_ORG test org login + * WARDEN_E2E_APPLY=1 (optional) also run the mutating Phase 2 + * + * ## Phase 1 — read-only contract checks (always, when configured) + * For every registered cycle: run `fetchLive` against the real org, then + * `buildDesired` + `diff`, and assert (a) every HTTP call was a GET — fetchLive + * never mutates — and (b) the pipeline composes into a valid change set. This + * is what catches GitHub API-contract drift (renamed fields, moved paths, + * permission changes), especially for the App-only token cycles that mocks + * can't validate. + * + * ## Phase 2 — one teardown-guarded mutation (only with WARDEN_E2E_APPLY=1) + * A single self-cleaning round-trip (create then delete a repo Actions + * variable) to prove the apply/write path works against real GitHub. + */ + +import { describe, it, beforeAll, expect } from "vitest"; +import { createAppClient, type AppClient } from "../src/auth/app-client.js"; +import { CYCLE_REGISTRY } from "../src/cli/registry.js"; +import { diff } from "../src/reconcile/diff.js"; +import type { RateBudget } from "../src/reconcile/runner.js"; +import type { OrgConfig, RepoConfig } from "../src/config/types.js"; + +// --------------------------------------------------------------------------- +// Gating +// --------------------------------------------------------------------------- + +const ENV = process.env; +const APP_ID = ENV.WARDEN_E2E_APP_ID; +const INSTALLATION_ID = ENV.WARDEN_E2E_INSTALLATION_ID; +const PRIVATE_KEY = ENV.WARDEN_E2E_PRIVATE_KEY?.replace(/\\n/g, "\n"); +const ORG = ENV.WARDEN_E2E_ORG; +const APPLY = ENV.WARDEN_E2E_APPLY === "1"; + +const configured = Boolean(APP_ID && INSTALLATION_ID && PRIVATE_KEY && ORG); +const suite = configured ? describe : describe.skip; + +if (!configured) { + // eslint-disable-next-line no-console + console.warn( + "[e2e] skipped — set WARDEN_E2E_APP_ID / _INSTALLATION_ID / _PRIVATE_KEY / _ORG to run.", + ); +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeBudget(initial = 500): RateBudget { + let remaining = initial; + return { + get remaining() { + return remaining; + }, + get exhausted() { + return remaining <= 0; + }, + use(n = 1) { + remaining = Math.max(0, remaining - n); + }, + }; +} + +interface Call { + method: string; + path: string; +} + +/** Wrap a client to record every (method, path) it is asked to perform. */ +function recording(inner: AppClient): { client: AppClient; calls: Call[] } { + const calls: Call[] = []; + return { + calls, + client: { + async request(method: string, path: string, body?: unknown): Promise { + calls.push({ method, path }); + return inner.request(method, path, body); + }, + }, + }; +} + +// --------------------------------------------------------------------------- +// Suite +// --------------------------------------------------------------------------- + +suite("warden e2e (real GitHub org)", () => { + let client: AppClient; + let scope: { repos: Record }; + let orgConfig: OrgConfig; + + beforeAll(async () => { + client = createAppClient({ + appId: APP_ID!, + installationId: INSTALLATION_ID!, + privateKeyPem: PRIVATE_KEY!, + }); + + // Discover a few repos so repo-scoped cycles have something to fetch. + const repos = await client.request>( + "GET", + `/orgs/${ORG}/repos?per_page=3&type=all`, + ); + const repoNames = (repos ?? []).map((r) => r.name).slice(0, 3); + + // A "kitchen-sink" repo config so every repo-scoped cycle's fetchLive + // actually hits its endpoints (all reads tolerate 404 for absent resources). + const repoCfg: RepoConfig = { + branchProtection: [{ pattern: "main" }], + security: { secretScanning: true }, + environments: [{ name: "production" }], + rulesets: [{ name: "warden-e2e-probe" }], + secrets: [{ name: "WARDEN_E2E_PROBE" }], + variables: [{ name: "WARDEN_E2E_PROBE" }], + dependabot: { content: "version: 2\nupdates: []\n" }, + description: "warden e2e (not written in Phase 1)", + }; + + const repoMap: Record = {}; + for (const n of repoNames) repoMap[n] = { ...repoCfg }; + + scope = { repos: repoMap }; + orgConfig = { + settings: {}, + rulesets: [], + tokenPolicy: { revokeExpired: true }, + tokenApproval: { default: "manual" }, + repos: repoMap, + }; + }, 60_000); + + // ── Phase 1: every cycle's read path is contract-valid and read-only ────── + + for (const cycle of Object.values(CYCLE_REGISTRY)) { + it(`${cycle.name}: fetchLive is read-only and diffs cleanly`, async () => { + const rec = recording(client); + const budget = makeBudget(); + + const live = await cycle.fetchLive(rec.client, ORG!, scope, budget); + const desired = cycle.buildDesired(orgConfig, ORG!, scope); + const changeSet = diff(ORG!, desired, live, {}); + + // fetchLive must never mutate — every call it made is a GET. + const nonGet = rec.calls.filter((c) => c.method !== "GET"); + expect(nonGet, `non-GET calls from ${cycle.name}.fetchLive`).toEqual([]); + + // The pipeline composed into a valid change set. + expect(Array.isArray(changeSet.entries)).toBe(true); + }, 60_000); + } + + // ── Phase 2: one teardown-guarded mutation (opt-in) ─────────────────────── + + (APPLY ? it : it.skip)( + "apply round-trip: create + delete a repo Actions variable", + async () => { + const repo = Object.keys(scope.repos)[0]; + expect(repo, "need at least one discovered repo").toBeTruthy(); + const base = `/repos/${ORG}/${repo}/actions/variables`; + const name = "WARDEN_E2E_PROBE"; + + try { + await client.request("POST", base, { name, value: "ok" }); + const got = await client.request<{ name: string; value: string }>("GET", `${base}/${name}`); + expect(got.value).toBe("ok"); + } finally { + // Always clean up, even if an assertion above failed. + await client.request("DELETE", `${base}/${name}`).catch(() => undefined); + } + }, + 60_000, + ); +}); diff --git a/package.json b/package.json index 26a61d3..1259cd7 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "scripts": { "tsc": "tsc --noEmit", "test": "vitest run", + "test:e2e": "vitest run --config vitest.e2e.config.ts", "build": "esbuild src/cli.ts --bundle --platform=node --format=esm --outfile=dist/cli.js && chmod +x dist/cli.js", "build:action": "esbuild src/action.ts --bundle --platform=node --format=esm --define:process.env.GITHUB_WARDEN_IS_ACTION='\"1\"' --outfile=action/index.mjs", "prepublishOnly": "npm run build" diff --git a/tsconfig.json b/tsconfig.json index cd2fe31..31b56be 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,6 +11,6 @@ "allowImportingTsExtensions": true, "verbatimModuleSyntax": false }, - "include": ["src/**/*"], + "include": ["src/**/*", "e2e/**/*"], "exclude": ["node_modules", "dist"] } diff --git a/vitest.e2e.config.ts b/vitest.e2e.config.ts new file mode 100644 index 0000000..23431cd --- /dev/null +++ b/vitest.e2e.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "vitest/config"; + +// Separate config for the gated end-to-end suite (real GitHub org). Kept out of +// the default `npm test` run, which only globs `src/**`. Run with +// `npm run test:e2e`; the suite self-skips unless the WARDEN_E2E_* env vars are +// set. A generous timeout absorbs real network latency / pagination. +export default defineConfig({ + test: { + include: ["e2e/**/*.e2e.test.ts"], + testTimeout: 60_000, + hookTimeout: 60_000, + }, +});