diff --git a/.github/workflows/ci-triage.yml b/.github/workflows/ci-triage.yml new file mode 100644 index 000000000..d6e72c8cc --- /dev/null +++ b/.github/workflows/ci-triage.yml @@ -0,0 +1,138 @@ +# CI failure triage. When CI fails on a pull request, post ONE comment that +# classifies the failure so attention is not spent on failures that are not the +# author's fault: +# - main-side: the same job is also failing on the latest default-branch run +# (CI merges the PR branch with current main, so a main regression surfaces on +# every open PR — this has cost debugging time before). +# - possible known flake: a UI/Playwright job failed — check tests/flake-ledger.json. +# - needs investigation: everything else. +# +# SHIPPED INERT: this event-triggered workflow does nothing until the repo variable +# CI_TRIAGE_ENABLED == "true". It never runs PR-authored code — it only reads job +# metadata + the committed flake ledger via the trusted default-branch checkout, and +# posts a comment with the built-in token. +name: CI Triage + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + +concurrency: + group: ci-triage-${{ github.event.workflow_run.id }} + cancel-in-progress: false + +permissions: + contents: read + actions: read + pull-requests: write + +jobs: + triage: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + if: > + vars.CI_TRIAGE_ENABLED == 'true' && + github.event.workflow_run.conclusion == 'failure' && + github.event.workflow_run.event == 'pull_request' + steps: + - name: Checkout default branch (trusted) for the flake ledger + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Post triage comment + uses: actions/github-script@v9 + with: + script: | + const fs = require("fs"); + const run = context.payload.workflow_run; + + // Resolve the PR for this run (empty for fork PRs → skip quietly). + const pr = (run.pull_requests || [])[0]; + if (!pr) { + core.info("No associated PR on the workflow_run payload; skipping."); + return; + } + + // Failed jobs for this run. + const jobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + }); + const failed = jobs.filter((j) => j.conclusion === "failure").map((j) => j.name); + if (failed.length === 0) { + core.info("No failed jobs found; skipping."); + return; + } + + // Is the same job failing on the latest default-branch CI run? → main-side. + let mainFailingJobs = new Set(); + try { + const { data: mainRuns } = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + event: "push", + branch: context.payload.repository.default_branch, + per_page: 1, + }); + // reuse: find the CI workflow's latest main run + const latestMain = (mainRuns.workflow_runs || []).find((r) => r.name === run.name); + if (latestMain && latestMain.conclusion === "failure") { + const mainJobs = await github.paginate(github.rest.actions.listJobsForWorkflowRun, { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: latestMain.id, + per_page: 100, + }); + mainFailingJobs = new Set(mainJobs.filter((j) => j.conclusion === "failure").map((j) => j.name)); + } + } catch (e) { + core.info(`main-side check skipped: ${e.message}`); + } + + const flakes = JSON.parse(fs.readFileSync("tests/flake-ledger.json", "utf8")).flakes || []; + const uiFlakeSpecs = flakes.map((f) => f.spec).filter((s) => /ui-/.test(s)); + const looksUi = (name) => /ui|playwright|browser|e2e/i.test(name); + + const lines = failed.map((name) => { + if (mainFailingJobs.has(name)) return `- \`${name}\` — **main-side**: also failing on the latest \`main\` run, likely not your change.`; + if (looksUi(name)) return `- \`${name}\` — **possible known flake**: UI/Playwright job; check \`tests/flake-ledger.json\`${uiFlakeSpecs.length ? ` (${uiFlakeSpecs.join(", ")})` : ""} and re-run before bisecting.`; + return `- \`${name}\` — **needs investigation**.`; + }); + + const marker = ""; + const body = [ + marker, + `### CI triage`, + `CI failed on this PR. Automated classification of the ${failed.length} failed job(s):`, + "", + ...lines, + "", + "_Heuristic only — a main-side or flake label is a starting point, not a verdict._", + ].join("\n"); + + // Replace a prior triage comment instead of stacking. + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + }); + const prior = comments.find((c) => (c.body || "").startsWith(marker)); + if (prior) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: prior.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pr.number, + body, + }); + } diff --git a/.github/workflows/ingestion-autopilot.yml b/.github/workflows/ingestion-autopilot.yml new file mode 100644 index 000000000..e7e1ec479 --- /dev/null +++ b/.github/workflows/ingestion-autopilot.yml @@ -0,0 +1,120 @@ +# Ingestion autopilot. Probes the live ingestion queue (reindex-health) and, when +# a stuck-queue signal is present, runs the existing recovery path. Alerts (opens/ +# updates an issue) only when Supabase is unreachable or recovery fails. +# +# SHIPPED DISABLED: workflow_dispatch only; the schedule is commented out. Recovery +# is DRY-RUN unless the run is explicitly told to apply (dispatch input `apply=true`, +# allowed only when repo variable INGESTION_AUTOPILOT_APPLY == "true"). To enable the +# cadence: set the repo secret below, confirm one dispatch dry-run, then uncomment +# `schedule:` and (optionally) set INGESTION_AUTOPILOT_APPLY=true. +name: Ingestion Autopilot + +on: + workflow_dispatch: + inputs: + apply: + description: "Apply recovery (requires repo var INGESTION_AUTOPILOT_APPLY=true)" + required: false + default: "false" + # schedule: + # # Every 6 hours. + # - cron: "0 */6 * * *" + +concurrency: + group: ingestion-autopilot + cancel-in-progress: false + +permissions: + contents: read + issues: write + +env: + NEXT_PUBLIC_SUPABASE_URL: https://sjrfecxgysukkwxsowpy.supabase.co + SUPABASE_PROJECT_REF: sjrfecxgysukkwxsowpy + SUPABASE_PROJECT_NAME: Clinical KB Database + NEXT_PUBLIC_SUPABASE_PUBLISHABLE_KEY: placeholder-ci-anon-key + SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SUPABASE_SERVICE_ROLE_KEY }} + +jobs: + ingestion-autopilot: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + + - name: Preflight required secrets + run: | + if [ -z "$SUPABASE_SERVICE_ROLE_KEY" ]; then + echo "::error::Ingestion autopilot cannot run — missing repo secret SUPABASE_SERVICE_ROLE_KEY" + exit 1 + fi + + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Guard Supabase project identity + run: npm run check:supabase-project + + - name: Run autopilot + id: autopilot + env: + # Apply only when BOTH the dispatch input and the repo variable allow it. + APPLY_REQUESTED: ${{ github.event.inputs.apply }} + APPLY_ALLOWED: ${{ vars.INGESTION_AUTOPILOT_APPLY }} + run: | + if [ "$APPLY_REQUESTED" = "true" ] && [ "$APPLY_ALLOWED" = "true" ]; then + echo "Applying recovery when stuck." + npm run ingestion:autopilot -- --apply + else + echo "Dry-run (set repo var INGESTION_AUTOPILOT_APPLY=true and dispatch apply=true to recover)." + npm run ingestion:autopilot + fi + + - name: Open or update alert issue + if: failure() && github.event_name == 'schedule' + uses: actions/github-script@v9 + with: + script: | + const label = "ingestion-autopilot"; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = [ + `Ingestion autopilot failed on ${new Date().toISOString()}.`, + "", + `Run: ${runUrl}`, + "", + "Either Supabase was unreachable or recovery failed. Triage:", + "`npm run reindex:health`, then `npm run recover:ingestion -- --apply` if a stuck", + "queue is confirmed. Do not assume corruption — a transient outage looks the same.", + ].join("\n"); + const { data: existing } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: "open", + labels: label, + }); + if (existing.length > 0) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing[0].number, + body, + }); + } else { + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: "Ingestion autopilot alert", + labels: [label], + body, + }); + } diff --git a/package.json b/package.json index b34baaebf..502762cb1 100644 --- a/package.json +++ b/package.json @@ -88,6 +88,8 @@ "medications:seed": "node scripts/run-tsx.mjs scripts/seed-medication-records.ts", "reindex": "node scripts/run-tsx.mjs scripts/reindex.ts", "reindex:health": "node scripts/run-tsx.mjs scripts/reindex-health.ts", + "ingestion:autopilot": "node scripts/run-tsx.mjs scripts/ingestion-autopilot.ts", + "flake:ledger": "node scripts/flake-ledger.mjs", "reindex:cleanup-staged": "node scripts/run-tsx.mjs scripts/cleanup-abandoned-reindex-generations.ts", "images:re-stamp-generation": "node scripts/run-tsx.mjs scripts/reindex-image-generation-metadata.ts", "supabase:recovery-status": "node scripts/run-tsx.mjs scripts/supabase-recovery-status.ts", diff --git a/scripts/flake-ledger.mjs b/scripts/flake-ledger.mjs new file mode 100644 index 000000000..3f615e6bc --- /dev/null +++ b/scripts/flake-ledger.mjs @@ -0,0 +1,77 @@ +#!/usr/bin/env node +/** + * flake-ledger — loader + matcher for the known-flaky Playwright specs recorded in + * tests/flake-ledger.json. + * + * Purpose: stop re-diagnosing the same flakes from memory every time CI goes red. + * The CI failure-triage workflow uses isKnownFlake() to attribute a failed test to + * a known flake; a serial re-run of only-flaky failures can be layered on top. + * + * CLI: + * node scripts/flake-ledger.mjs --list print the ledger + * node scripts/flake-ledger.mjs --self-test validate shape + matcher + * node scripts/flake-ledger.mjs --match "" → prints matching id or "none" + */ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const LEDGER_PATH = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "flake-ledger.json"); + +export function loadFlakeLedger(ledgerPath = LEDGER_PATH) { + const raw = JSON.parse(readFileSync(ledgerPath, "utf8")); + const flakes = Array.isArray(raw.flakes) ? raw.flakes : []; + for (const flake of flakes) { + if (!flake.id || !flake.match || !flake.spec || !flake.reason) { + throw new Error(`flake-ledger entry missing required field (id/match/spec/reason): ${JSON.stringify(flake)}`); + } + } + return flakes; +} + +/** Return the matching flake entry for a test title, or null. Case-insensitive substring. */ +export function matchFlake(testTitle, flakes = loadFlakeLedger()) { + if (!testTitle) return null; + const haystack = String(testTitle).toLowerCase(); + return flakes.find((flake) => haystack.includes(String(flake.match).toLowerCase())) ?? null; +} + +export function isKnownFlake(testTitle, flakes = loadFlakeLedger()) { + return matchFlake(testTitle, flakes) !== null; +} + +function selfTest() { + const flakes = loadFlakeLedger(); + const assert = (cond, label) => { + if (!cond) { + console.error(`✖ self-test failed: ${label}`); + process.exitCode = 1; + throw new Error(label); + } + }; + assert(flakes.length > 0, "ledger is non-empty"); + const ids = new Set(flakes.map((f) => f.id)); + assert(ids.size === flakes.length, "flake ids are unique"); + assert(isKnownFlake("composer hero renders on hydrate", flakes), "matches a known flake by title substring"); + assert(!isKnownFlake("a totally unrelated passing test", flakes), "does not match an unrelated title"); + assert(matchFlake("", flakes) === null, "empty title matches nothing"); + if (process.exitCode !== 1) console.error("[flake-ledger] self-test passed"); +} + +function main() { + if (process.argv.includes("--self-test")) return selfTest(); + if (process.argv.includes("--list")) { + for (const flake of loadFlakeLedger()) console.log(`${flake.id}\t${flake.spec}\t"${flake.match}"`); + return; + } + const matchIndex = process.argv.indexOf("--match"); + if (matchIndex >= 0) { + const hit = matchFlake(process.argv[matchIndex + 1], loadFlakeLedger()); + console.log(hit ? hit.id : "none"); + return; + } + console.error('usage: flake-ledger.mjs [--list | --self-test | --match ""]'); +} + +const invokedDirectly = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (invokedDirectly) main(); diff --git a/scripts/ingestion-autopilot.ts b/scripts/ingestion-autopilot.ts new file mode 100644 index 000000000..120e70b35 --- /dev/null +++ b/scripts/ingestion-autopilot.ts @@ -0,0 +1,166 @@ +import { spawnSync } from "node:child_process"; + +// ingestion-autopilot — self-healing loop for a stuck ingestion queue. +// +// Silent ingestion stalls are invisible product damage: documents just quietly +// stop becoming searchable. This chains the existing read-only probe +// (reindex-health) to a decision, and — only when a stuck-queue signal is present +// and --apply is passed — runs the existing recovery path (recover-ingestion-queue +// --apply). It ALERTS (non-zero exit) only when Supabase is unreachable or recovery +// fails; a healthy or merely-idle queue exits 0. +// +// Decision is based purely on reindex-health's JSON (a clean read-only probe). +// check-indexing is intentionally NOT chained here: it additionally requires the +// OpenAI + Python/PDF stack and is a strict corpus-readiness gate, so folding it in +// would make the autopilot fragile and noisy. It remains the operator's separate +// readiness check. +// +// Dry-run by default (reports what it WOULD recover). Provider-touching, so it runs +// only in the disabled scheduled workflow or a manual operator run — never in PR CI. + +export type IngestionHealth = { + ok?: boolean; + status?: string; + counts?: Record<string, number | null>; + openJobs?: Array<{ + id?: string; + status?: string; + stage?: string | null; + attempt_count?: number | null; + max_attempts?: number | null; + locked_at?: string | null; + error_message?: string | null; + }>; +}; + +export type IngestionAssessment = { + available: boolean; + stuck: boolean; + reasons: string[]; + failedJobs: number; + staleProcessingJobs: number; +}; + +/** + * Pure assessment of a reindex-health JSON payload. `stuck` is true when there are + * failed jobs, any open job already in the `failed` state, or a `processing` job + * whose lock is older than staleAfterMinutes (a worker died mid-job). Env-free and + * unit-tested. + */ +export function assessIngestionHealth( + health: IngestionHealth, + options: { staleAfterMinutes?: number; now?: number } = {}, +): IngestionAssessment { + const staleAfterMinutes = options.staleAfterMinutes ?? 30; + const now = options.now ?? Date.now(); + + if (health.ok === false || health.status === "supabase_unavailable") { + return { available: false, stuck: false, reasons: ["supabase unavailable"], failedJobs: 0, staleProcessingJobs: 0 }; + } + + const reasons: string[] = []; + const failedJobs = Number(health.counts?.jobs_failed ?? 0) || 0; + if (failedJobs > 0) reasons.push(`${failedJobs} failed job(s)`); + + const openJobs = health.openJobs ?? []; + const staleThresholdMs = staleAfterMinutes * 60_000; + let staleProcessingJobs = 0; + for (const job of openJobs) { + if (job.status === "failed") continue; // already counted via jobs_failed + if (job.status === "processing" && job.locked_at) { + const lockedAt = Date.parse(job.locked_at); + if (Number.isFinite(lockedAt) && now - lockedAt > staleThresholdMs) staleProcessingJobs += 1; + } + } + if (staleProcessingJobs > 0) { + reasons.push(`${staleProcessingJobs} processing job(s) locked > ${staleAfterMinutes}m (stale worker)`); + } + + return { + available: true, + stuck: reasons.length > 0, + reasons, + failedJobs, + staleProcessingJobs, + }; +} + +function parseIntFlag(name: string, fallback: number): number { + const index = process.argv.indexOf(name); + if (index < 0) return fallback; + const value = Number.parseInt(process.argv[index + 1] ?? "", 10); + return Number.isFinite(value) && value > 0 ? value : fallback; +} + +function runTsxScript(script: string, args: string[] = []) { + return spawnSync(process.execPath, ["scripts/run-tsx.mjs", script, ...args], { + cwd: process.cwd(), + encoding: "utf8", + }); +} + +function main() { + const apply = process.argv.includes("--apply"); + const staleAfterMinutes = parseIntFlag("--stale-after-minutes", 30); + const limit = parseIntFlag("--limit", 20); + + const probe = runTsxScript("scripts/reindex-health.ts"); + const stdout = probe.stdout ?? ""; + process.stdout.write(stdout); + + let health: IngestionHealth; + try { + health = JSON.parse(stdout); + } catch { + console.error("[autopilot] could not parse reindex-health output — aborting"); + process.exitCode = 1; + return; + } + + const assessment = assessIngestionHealth(health, { staleAfterMinutes }); + if (!assessment.available) { + console.error("[autopilot] Supabase unavailable — not attempting recovery. Alerting."); + process.exitCode = 1; + return; + } + + if (!assessment.stuck) { + console.log("[autopilot] queue healthy — nothing to recover."); + return; + } + + console.log(`[autopilot] stuck-queue signal: ${assessment.reasons.join("; ")}`); + + if (!apply) { + console.log( + `[autopilot] DRY RUN — would run: npm run recover:ingestion -- --apply --yes ` + + `--stale-after-minutes ${staleAfterMinutes} --limit ${limit}\n` + + `[autopilot] pass --apply to recover.`, + ); + return; + } + + console.log("[autopilot] applying recovery…"); + const recover = runTsxScript("scripts/recover-ingestion-queue.ts", [ + "--apply", + "--yes", + "--stale-after-minutes", + String(staleAfterMinutes), + "--limit", + String(limit), + ]); + if (recover.stdout) process.stdout.write(recover.stdout); + if (recover.stderr) process.stderr.write(recover.stderr); + if ((recover.status ?? 1) !== 0) { + console.error("[autopilot] recovery FAILED — alerting."); + process.exitCode = 1; + return; + } + console.log("[autopilot] recovery applied. Re-probe with npm run reindex:health to confirm."); +} + +// Only run as a CLI when invoked directly — importing (tests) must not execute. +const invokedDirectly = process.argv[1]?.endsWith("ingestion-autopilot.ts"); +if (invokedDirectly) { + main(); +} diff --git a/tests/flake-ledger.json b/tests/flake-ledger.json new file mode 100644 index 000000000..0db897684 --- /dev/null +++ b/tests/flake-ledger.json @@ -0,0 +1,37 @@ +{ + "$comment": "Known-flaky Playwright specs. Durable knowledge so a failure gets attributed to a known flake instead of re-diagnosed from memory each time. Consumed by scripts/flake-ledger.mjs (loader + isKnownFlake) and the CI failure-triage workflow. `match` is a case-insensitive substring of the test title; keep entries narrow. Removing an entry means 'we believe this is fixed' — do that deliberately.", + "flakes": [ + { + "id": "raf-portal-hydration", + "match": "composer hero", + "spec": "tests/ui-smoke.spec.ts", + "reason": "Hero-composer hydration depends on a rAF portal that starves under a headless render; fixed with a MutationObserver microtask (#502/#504) but can still flake on a cold render.", + "since": "2026-07-11", + "refs": ["#502", "#504"] + }, + { + "id": "tap-target-subpixel", + "match": "tap target", + "spec": "tests/ui-smoke.spec.ts", + "reason": "min-h-11 tap-target rect rounds sub-pixel below 44px on some DPRs; mitigated by min-h-12 but historically flaky.", + "since": "2026-07-11", + "refs": [] + }, + { + "id": "narrow-viewport-parallel", + "match": "narrow viewport", + "spec": "tests/ui-tools.spec.ts", + "reason": "Differentials narrow-viewport assertions race only under parallel workers; the suite runs workers:1 but sharded CI can still interleave.", + "since": "2026-07-11", + "refs": [] + }, + { + "id": "rag-answer-fallback-timeout", + "match": "answer fallback", + "spec": "tests/ui-tools.spec.ts", + "reason": "The RAG answer-fallback path can exceed the 60s test timeout when the provider route is cold; a pre-existing advisory flake.", + "since": "2026-07-11", + "refs": [] + } + ] +} diff --git a/tests/flake-ledger.test.ts b/tests/flake-ledger.test.ts new file mode 100644 index 000000000..fd8dae544 --- /dev/null +++ b/tests/flake-ledger.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { isKnownFlake, loadFlakeLedger, matchFlake } from "../scripts/flake-ledger.mjs"; + +describe("flake ledger", () => { + it("loads and validates the committed ledger", () => { + const flakes = loadFlakeLedger(); + expect(flakes.length).toBeGreaterThan(0); + for (const flake of flakes) { + expect(flake.id).toBeTruthy(); + expect(flake.match).toBeTruthy(); + expect(flake.spec).toMatch(/^tests\//); + expect(flake.reason).toBeTruthy(); + } + }); + + it("has unique ids", () => { + const ids = loadFlakeLedger().map((f: { id: string }) => f.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + it("matches a known flake by case-insensitive title substring", () => { + expect(isKnownFlake("Composer Hero mounts after hydration")).toBe(true); + const hit = matchFlake("the tap target is at least 44px"); + expect(hit?.id).toBe("tap-target-subpixel"); + }); + + it("does not match an unrelated title or an empty string", () => { + expect(isKnownFlake("renders the medication results grid")).toBe(false); + expect(matchFlake("")).toBeNull(); + }); +}); diff --git a/tests/ingestion-autopilot.test.ts b/tests/ingestion-autopilot.test.ts new file mode 100644 index 000000000..a7123e2b7 --- /dev/null +++ b/tests/ingestion-autopilot.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from "vitest"; +import { assessIngestionHealth } from "../scripts/ingestion-autopilot"; + +const NOW = Date.parse("2026-07-13T12:00:00Z"); +const minutesAgo = (m: number) => new Date(NOW - m * 60_000).toISOString(); + +describe("assessIngestionHealth", () => { + it("reports unavailable (and not stuck) when Supabase is down", () => { + const a = assessIngestionHealth({ ok: false, status: "supabase_unavailable" }, { now: NOW }); + expect(a.available).toBe(false); + expect(a.stuck).toBe(false); + expect(a.reasons).toContain("supabase unavailable"); + }); + + it("is healthy when there are no failed or stale jobs", () => { + const a = assessIngestionHealth( + { ok: true, counts: { jobs_failed: 0 }, openJobs: [{ status: "pending" }] }, + { now: NOW }, + ); + expect(a.available).toBe(true); + expect(a.stuck).toBe(false); + expect(a.reasons).toHaveLength(0); + }); + + it("flags failed jobs as stuck", () => { + const a = assessIngestionHealth({ ok: true, counts: { jobs_failed: 3 }, openJobs: [] }, { now: NOW }); + expect(a.stuck).toBe(true); + expect(a.failedJobs).toBe(3); + expect(a.reasons[0]).toContain("3 failed job"); + }); + + it("flags a processing job whose lock is older than the stale threshold", () => { + const a = assessIngestionHealth( + { + ok: true, + counts: { jobs_failed: 0 }, + openJobs: [{ status: "processing", locked_at: minutesAgo(45) }], + }, + { now: NOW, staleAfterMinutes: 30 }, + ); + expect(a.stuck).toBe(true); + expect(a.staleProcessingJobs).toBe(1); + }); + + it("does not flag a recently-locked processing job", () => { + const a = assessIngestionHealth( + { + ok: true, + counts: { jobs_failed: 0 }, + openJobs: [{ status: "processing", locked_at: minutesAgo(5) }], + }, + { now: NOW, staleAfterMinutes: 30 }, + ); + expect(a.stuck).toBe(false); + expect(a.staleProcessingJobs).toBe(0); + }); +});