feat(ops): ingestion autopilot, flake ledger, and CI failure triage #588
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,138 @@ | ||
| # CI failure triage. When CI fails on a pull request, post ONE comment that | ||
| # classifies the failure so attention is not spent on failures that are not the | ||
| # author's fault: | ||
| # - main-side: the same job is also failing on the latest default-branch run | ||
| # (CI merges the PR branch with current main, so a main regression surfaces on | ||
| # every open PR — this has cost debugging time before). | ||
| # - possible known flake: a UI/Playwright job failed — check tests/flake-ledger.json. | ||
| # - needs investigation: everything else. | ||
| # | ||
| # SHIPPED INERT: this event-triggered workflow does nothing until the repo variable | ||
| # CI_TRIAGE_ENABLED == "true". It never runs PR-authored code — it only reads job | ||
| # metadata + the committed flake ledger via the trusted default-branch checkout, and | ||
| # posts a comment with the built-in token. | ||
| name: CI Triage | ||
|
|
||
| on: | ||
| workflow_run: | ||
| workflows: ["CI"] | ||
| types: [completed] | ||
|
|
||
| concurrency: | ||
| group: ci-triage-${{ github.event.workflow_run.id }} | ||
| cancel-in-progress: false | ||
|
|
||
| permissions: | ||
| contents: read | ||
| actions: read | ||
| pull-requests: write | ||
|
|
||
| jobs: | ||
| triage: | ||
| runs-on: ubuntu-24.04 | ||
| timeout-minutes: 10 | ||
| if: > | ||
| vars.CI_TRIAGE_ENABLED == 'true' && | ||
| github.event.workflow_run.conclusion == 'failure' && | ||
| github.event.workflow_run.event == 'pull_request' | ||
| steps: | ||
| - name: Checkout default branch (trusted) for the flake ledger | ||
| uses: actions/checkout@v6 | ||
| with: | ||
| persist-credentials: false | ||
|
|
||
| - name: Post triage comment | ||
| uses: actions/github-script@v9 | ||
| with: | ||
| script: | | ||
| const fs = require("fs"); | ||
| const run = context.payload.workflow_run; | ||
|
|
||
| // Resolve the PR for this run (empty for fork PRs → skip quietly). | ||
| const pr = (run.pull_requests || [])[0]; | ||
| if (!pr) { | ||
| core.info("No associated PR on the workflow_run payload; skipping."); | ||
| return; | ||
| } | ||
|
|
||
| // Failed jobs for this run. | ||
| const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| run_id: run.id, | ||
| per_page: 100, | ||
| }); | ||
| const failed = jobs.filter((j) => j.conclusion === "failure").map((j) => j.name); | ||
| if (failed.length === 0) { | ||
| core.info("No failed jobs found; skipping."); | ||
| return; | ||
| } | ||
|
|
||
| // Is the same job failing on the latest default-branch CI run? → main-side. | ||
| let mainFailingJobs = new Set(); | ||
| try { | ||
| const { data: mainRuns } = await github.rest.actions.listWorkflowRunsForRepo({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| event: "push", | ||
| branch: context.payload.repository.default_branch, | ||
| per_page: 1, | ||
| }); | ||
| // reuse: find the CI workflow's latest main run | ||
| const latestMain = (mainRuns.workflow_runs || []).find((r) => r.name === run.name); | ||
| if (latestMain && latestMain.conclusion === "failure") { | ||
| const mainJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| run_id: latestMain.id, | ||
| per_page: 100, | ||
| }); | ||
| mainFailingJobs = new Set(mainJobs.filter((j) => j.conclusion === "failure").map((j) => j.name)); | ||
| } | ||
| } catch (e) { | ||
| core.info(`main-side check skipped: ${e.message}`); | ||
| } | ||
|
|
||
| const flakes = JSON.parse(fs.readFileSync("tests/flake-ledger.json", "utf8")).flakes || []; | ||
| const uiFlakeSpecs = flakes.map((f) => f.spec).filter((s) => /ui-/.test(s)); | ||
| const looksUi = (name) => /ui|playwright|browser|e2e/i.test(name); | ||
|
|
||
| const lines = failed.map((name) => { | ||
| if (mainFailingJobs.has(name)) return `- \`${name}\` — **main-side**: also failing on the latest \`main\` run, likely not your change.`; | ||
| if (looksUi(name)) return `- \`${name}\` — **possible known flake**: UI/Playwright job; check \`tests/flake-ledger.json\`${uiFlakeSpecs.length ? ` (${uiFlakeSpecs.join(", ")})` : ""} and re-run before bisecting.`; | ||
| return `- \`${name}\` — **needs investigation**.`; | ||
| }); | ||
|
|
||
| const marker = "<!-- ci-triage -->"; | ||
| const body = [ | ||
| marker, | ||
| `### CI triage`, | ||
| `CI failed on this PR. Automated classification of the ${failed.length} failed job(s):`, | ||
| "", | ||
| ...lines, | ||
| "", | ||
| "_Heuristic only — a main-side or flake label is a starting point, not a verdict._", | ||
| ].join("\n"); | ||
|
|
||
| // Replace a prior triage comment instead of stacking. | ||
| const { data: comments } = await github.rest.issues.listComments({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: pr.number, | ||
| }); | ||
| const prior = comments.find((c) => (c.body || "").startsWith(marker)); | ||
| if (prior) { | ||
| await github.rest.issues.updateComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| comment_id: prior.id, | ||
| body, | ||
| }); | ||
| } else { | ||
| await github.rest.issues.createComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: pr.number, | ||
| body, | ||
| }); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| # Ingestion autopilot. Probes the live ingestion queue (reindex-health) and, when | ||
| # a stuck-queue signal is present, runs the existing recovery path. Alerts (opens/ | ||
| # updates an issue) only when Supabase is unreachable or recovery fails. | ||
| # | ||
| # SHIPPED DISABLED: workflow_dispatch only; the schedule is commented out. Recovery | ||
| # is DRY-RUN unless the run is explicitly told to apply (dispatch input `apply=true`, | ||
| # allowed only when repo variable INGESTION_AUTOPILOT_APPLY == "true"). To enable the | ||
| # cadence: set the repo secret below, confirm one dispatch dry-run, then uncomment | ||
| # `schedule:` and (optionally) set INGESTION_AUTOPILOT_APPLY=true. | ||
| name: Ingestion Autopilot | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| apply: | ||
| description: "Apply recovery (requires repo var INGESTION_AUTOPILOT_APPLY=true)" | ||
| required: false | ||
| default: "false" | ||
| # schedule: | ||
| # # Every 6 hours. | ||
| # - cron: "0 */6 * * *" | ||
|
|
||
| concurrency: | ||
| group: ingestion-autopilot | ||
| cancel-in-progress: false | ||
|
|
||
| permissions: | ||
| contents: read | ||
| issues: write | ||
|
|
||
| env: | ||
| NEXT_PUBLIC_SUPABASE_URL: https://sjrfecxgysukkwxsowpy.supabase.co | ||
| SUPABASE_PROJECT_REF: sjrfecxgysukkwxsowpy | ||
| SUPABASE_PROJECT_NAME: Clinical KB Database | ||
| NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key | ||
| SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} | ||
|
|
||
| jobs: | ||
| ingestion-autopilot: | ||
| runs-on: ubuntu-24.04 | ||
| timeout-minutes: 20 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v6 | ||
| with: | ||
| persist-credentials: false | ||
|
|
||
| - name: Preflight required secrets | ||
| run: | | ||
| if [ -z "$SUPABASE_SERVICE_ROLE_KEY" ]; then | ||
| echo "::error::Ingestion autopilot cannot run — missing repo secret SUPABASE_SERVICE_ROLE_KEY" | ||
| exit 1 | ||
| fi | ||
|
|
||
| - name: Setup Node.js | ||
| uses: actions/setup-node@v5 | ||
| with: | ||
| node-version-file: ".nvmrc" | ||
| cache: npm | ||
| cache-dependency-path: package-lock.json | ||
|
|
||
| - name: Install dependencies | ||
| run: npm ci | ||
|
|
||
| - name: Guard Supabase project identity | ||
| run: npm run check:supabase-project | ||
|
|
||
| - name: Run autopilot | ||
| id: autopilot | ||
| env: | ||
| # Apply only when BOTH the dispatch input and the repo variable allow it. | ||
| APPLY_REQUESTED: ${{ github.event.inputs.apply }} | ||
| APPLY_ALLOWED: ${{ vars.INGESTION_AUTOPILOT_APPLY }} | ||
| run: | | ||
| if [ "$APPLY_REQUESTED" = "true" ] && [ "$APPLY_ALLOWED" = "true" ]; then | ||
|
Comment on lines
+72
to
+75
There was a problem hiding this comment.
When the commented Useful? React with 👍 / 👎. |
||
| echo "Applying recovery when stuck." | ||
| npm run ingestion:autopilot -- --apply | ||
| else | ||
| echo "Dry-run (set repo var INGESTION_AUTOPILOT_APPLY=true and dispatch apply=true to recover)." | ||
| npm run ingestion:autopilot | ||
| fi | ||
|
|
||
| - name: Open or update alert issue | ||
| if: failure() && github.event_name == 'schedule' | ||
| uses: actions/github-script@v9 | ||
| with: | ||
| script: | | ||
| const label = "ingestion-autopilot"; | ||
| const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; | ||
| const body = [ | ||
| `Ingestion autopilot failed on ${new Date().toISOString()}.`, | ||
| "", | ||
| `Run: ${runUrl}`, | ||
| "", | ||
| "Either Supabase was unreachable or recovery failed. Triage:", | ||
| "`npm run reindex:health`, then `npm run recover:ingestion -- --apply` if a stuck", | ||
| "queue is confirmed. Do not assume corruption — a transient outage looks the same.", | ||
| ].join("\n"); | ||
| const { data: existing } = await github.rest.issues.listForRepo({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| state: "open", | ||
| labels: label, | ||
| }); | ||
| if (existing.length > 0) { | ||
| await github.rest.issues.createComment({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| issue_number: existing[0].number, | ||
| body, | ||
| }); | ||
| } else { | ||
| await github.rest.issues.create({ | ||
| owner: context.repo.owner, | ||
| repo: context.repo.repo, | ||
| title: "Ingestion autopilot alert", | ||
| labels: [label], | ||
| body, | ||
| }); | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| #!/usr/bin/env node | ||
| /** | ||
| * flake-ledger — loader + matcher for the known-flaky Playwright specs recorded in | ||
| * tests/flake-ledger.json. | ||
| * | ||
| * Purpose: stop re-diagnosing the same flakes from memory every time CI goes red. | ||
| * The CI failure-triage workflow uses isKnownFlake() to attribute a failed test to | ||
| * a known flake; a serial re-run of only-flaky failures can be layered on top. | ||
| * | ||
| * CLI: | ||
| * node scripts/flake-ledger.mjs --list print the ledger | ||
| * node scripts/flake-ledger.mjs --self-test validate shape + matcher | ||
| * node scripts/flake-ledger.mjs --match "<test title>" → prints matching id or "none" | ||
| */ | ||
| import { readFileSync } from "node:fs"; | ||
| import path from "node:path"; | ||
| import { fileURLToPath, pathToFileURL } from "node:url"; | ||
|
|
||
| const LEDGER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "flake-ledger.json"); | ||
|
|
||
| export function loadFlakeLedger(ledgerPath = LEDGER_PATH) { | ||
| const raw = JSON.parse(readFileSync(ledgerPath, "utf8")); | ||
| const flakes = Array.isArray(raw.flakes) ? raw.flakes : []; | ||
| for (const flake of flakes) { | ||
| if (!flake.id || !flake.match || !flake.spec || !flake.reason) { | ||
| throw new Error(`flake-ledger entry missing required field (id/match/spec/reason): ${JSON.stringify(flake)}`); | ||
| } | ||
| } | ||
| return flakes; | ||
| } | ||
|
|
||
| /** Return the matching flake entry for a test title, or null. Case-insensitive substring. */ | ||
| export function matchFlake(testTitle, flakes = loadFlakeLedger()) { | ||
| if (!testTitle) return null; | ||
| const haystack = String(testTitle).toLowerCase(); | ||
| return flakes.find((flake) => haystack.includes(String(flake.match).toLowerCase())) ?? null; | ||
| } | ||
|
|
||
| export function isKnownFlake(testTitle, flakes = loadFlakeLedger()) { | ||
| return matchFlake(testTitle, flakes) !== null; | ||
| } | ||
|
|
||
| function selfTest() { | ||
| const flakes = loadFlakeLedger(); | ||
| const assert = (cond, label) => { | ||
| if (!cond) { | ||
| console.error(`✖ self-test failed: ${label}`); | ||
| process.exitCode = 1; | ||
| throw new Error(label); | ||
| } | ||
| }; | ||
| assert(flakes.length > 0, "ledger is non-empty"); | ||
| const ids = new Set(flakes.map((f) => f.id)); | ||
| assert(ids.size === flakes.length, "flake ids are unique"); | ||
| assert(isKnownFlake("composer hero renders on hydrate", flakes), "matches a known flake by title substring"); | ||
| assert(!isKnownFlake("a totally unrelated passing test", flakes), "does not match an unrelated title"); | ||
| assert(matchFlake("", flakes) === null, "empty title matches nothing"); | ||
| if (process.exitCode !== 1) console.error("[flake-ledger] self-test passed"); | ||
| } | ||
|
|
||
| function main() { | ||
| if (process.argv.includes("--self-test")) return selfTest(); | ||
| if (process.argv.includes("--list")) { | ||
| for (const flake of loadFlakeLedger()) console.log(`${flake.id}\t${flake.spec}\t"${flake.match}"`); | ||
| return; | ||
| } | ||
| const matchIndex = process.argv.indexOf("--match"); | ||
| if (matchIndex >= 0) { | ||
| const hit = matchFlake(process.argv[matchIndex + 1], loadFlakeLedger()); | ||
| console.log(hit ? hit.id : "none"); | ||
| return; | ||
| } | ||
| console.error('usage: flake-ledger.mjs [--list | --self-test | --match "<title>"]'); | ||
| } | ||
|
|
||
| const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; | ||
| if (invokedDirectly) main(); |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This fetches only the single latest default-branch workflow run across the repository, then later looks for one whose name matches
CI. If any other workflow ran onmainmore recently than CI,latestMainis undefined andmainFailingJobsstays empty, so PR failures caused by an already-broken main are mislabeled as flakes or needing investigation. Query the specific workflow or fetch enough runs before selecting the latest matching CI run.Useful? React with 👍 / 👎.