From 6765472754611a6f48e1b6c35ed2a48a86526539 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 25 Apr 2026 10:37:49 -0400 Subject: [PATCH] feat(agents): triage recovery sweeps + local fire script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent triage-routine resilience improvements that share the same theme (close the silent-failure modes): 1. **Clear stuck claude-triaging labels** workflow. Runs every 30 min, removes `claude-triaging` from issues where the label has been on for >30 min — that's the orphan-label case where the routine errored mid-run and never swapped the label for `claude-triaged`. Without this, orphaned labels poison the "is anyone working on this?" signal. 2. **Triage webhook-miss sweep** workflow. Runs hourly, finds issues opened in the last 24h with no `claude-triag*` label and no `## Triage` comment, fires the routine manually with a `RECOVERY SWEEP:` payload tag. Closes the silent-failure mode that hit #3112 — webhook drops mean issues sit unprocessed forever with no audit trail. 3. **`.agents/scripts/triage-local.sh`**. Local CLI that POSTs to the routine's /fire endpoint with an issue payload, bypassing GitHub webhooks entirely. Useful when you need to fire triage immediately without leaving a public /triage comment trail, or when you don't want to wait an hour for the sweep. Reads `CLAUDE_ROUTINE_TRIAGE_URL` / `_TOKEN` from env or a local `.env`. Writes nothing to GitHub directly — the routine does all the comment/label work on its own. Playbook updated to document all three recovery paths under the existing "Triage Routine — Manual Nudge" section. Co-Authored-By: Claude Opus 4.7 (1M context) --- .agents/playbook.md | 28 +++- .agents/scripts/triage-local.sh | 138 +++++++++++++++++ .changeset/triage-recovery-and-local.md | 3 + .../workflows/clear-stuck-claude-triaging.yml | 78 ++++++++++ .../workflows/triage-webhook-miss-sweep.yml | 140 ++++++++++++++++++ 5 files changed, 385 insertions(+), 2 deletions(-) create mode 100755 .agents/scripts/triage-local.sh create mode 100644 .changeset/triage-recovery-and-local.md create mode 100644 .github/workflows/clear-stuck-claude-triaging.yml create mode 100644 .github/workflows/triage-webhook-miss-sweep.yml diff --git a/.agents/playbook.md b/.agents/playbook.md index e287c82f2b..f1125df1d4 100644 --- a/.agents/playbook.md +++ b/.agents/playbook.md @@ -449,8 +449,32 @@ lands on an open issue. To poke the routine yourself: usual cause. Comment `/triage` to recover. If `claude-triaging` is stuck on an issue for >30 minutes with no -visible progress, the routine errored mid-run. Remove the label -manually and re-fire with `/triage`. +visible progress, the routine errored mid-run. The +`Clear stuck claude-triaging labels` workflow runs every 30 minutes +and clears the label automatically; if you need it gone faster, +remove it manually and re-fire with `/triage`. + +**Recovery against silent webhook misses.** GitHub occasionally +drops `issues.opened` webhook deliveries with no audit trail (this +is what bit #3112 — issue created, no triage workflow run, no +signal). The `Triage webhook-miss sweep` workflow runs hourly, finds +issues opened in the last 24h with no `claude-triag*` label and no +`## Triage` comment, and fires the routine manually as a recovery. +Tag-line in the payload is `RECOVERY SWEEP:` so the routine knows +this is a catch-up rather than a fresh fire. + +**Local manual fire.** For "I need to fire triage right now without +leaving a public `/triage` comment trail" or "the sweep won't pick +this up for an hour" cases, use `.agents/scripts/triage-local.sh`: + +```bash +.agents/scripts/triage-local.sh [execute|clarify|defer] +``` + +Requires `CLAUDE_ROUTINE_TRIAGE_URL` and `CLAUDE_ROUTINE_TRIAGE_TOKEN` +env vars (or a local `.env` file). The script writes nothing to +GitHub — it only POSTs to the routine's `/fire` endpoint with the +issue payload. The routine itself does all the comment/label work. ## Cross-Agent Integration diff --git a/.agents/scripts/triage-local.sh b/.agents/scripts/triage-local.sh new file mode 100755 index 0000000000..02a433b413 --- /dev/null +++ b/.agents/scripts/triage-local.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Manually fire the Claude Issue Triage routine on an issue, bypassing +# GitHub webhooks. Useful when the webhook delivery missed (silent failure) +# or when you want to nudge the routine without leaving a public `/triage` +# comment trail. +# +# Usage: +# .agents/scripts/triage-local.sh [execute|clarify|defer] +# +# Examples: +# .agents/scripts/triage-local.sh 3112 # fresh triage +# .agents/scripts/triage-local.sh 3112 execute # bias toward Execute +# .agents/scripts/triage-local.sh 3112 clarify # force clarify +# +# Required env vars (or .env file in the cwd): +# CLAUDE_ROUTINE_TRIAGE_URL — full /fire URL for the routine +# CLAUDE_ROUTINE_TRIAGE_TOKEN — bearer token for that routine +# +# Both can be loaded from a local .env, op:// references resolved with +# `op run`, or your shell rc. The script does NOT touch GitHub repo state +# beyond reading the issue — no comments, no labels are written by this +# script; the routine itself does that on the GitHub side. + +set -euo pipefail + +ISSUE_NUM="${1:-}" +MODIFIER="${2:-}" + +if [ -z "$ISSUE_NUM" ]; then + echo "usage: $(basename "$0") [execute|clarify|defer]" >&2 + exit 64 +fi + +# Optional .env loader. +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + . .env + set +a +fi + +: "${CLAUDE_ROUTINE_TRIAGE_URL:?env var CLAUDE_ROUTINE_TRIAGE_URL must be set}" +: "${CLAUDE_ROUTINE_TRIAGE_TOKEN:?env var CLAUDE_ROUTINE_TRIAGE_TOKEN must be set}" + +if ! command -v gh >/dev/null 2>&1; then + echo "error: gh CLI not found" >&2 + exit 1 +fi +if ! command -v jq >/dev/null 2>&1; then + echo "error: jq not found" >&2 + exit 1 +fi + +REPO=$(gh repo view --json owner,name --jq '.owner.login + "/" + .name') +echo "Repo: $REPO" +echo "Firing triage for issue #$ISSUE_NUM${MODIFIER:+ (modifier: /$MODIFIER)}" + +issue=$(gh api "repos/$REPO/issues/$ISSUE_NUM") +title=$(echo "$issue" | jq -r '.title') +body=$(echo "$issue" | jq -r '.body // ""') +author=$(echo "$issue" | jq -r '.user.login') +assoc=$(echo "$issue" | jq -r '.author_association // "NONE"') +labels=$(echo "$issue" | jq -c '[.labels[].name]') +html_url=$(echo "$issue" | jq -r '.html_url') + +body_safe=$(printf '%s' "$body" | tr -d '\000' | head -c 8192) + +if [ -n "$MODIFIER" ]; then + case "$MODIFIER" in + execute|clarify|defer) + ;; + *) + echo "error: modifier must be one of: execute, clarify, defer (got: $MODIFIER)" >&2 + exit 64 + ;; + esac + nudge="MANUAL NUDGE: triage-local.sh requested triage with /$MODIFIER. Treat as an explicit request; skip already-engaged check. Honor the modifier (execute / clarify / defer)." + kind="manual" + action="triage" +else + nudge="" + kind="auto" + action="opened" +fi + +payload=$(jq -n \ + --arg repo "$REPO" \ + --arg num "$ISSUE_NUM" \ + --arg title "$title" \ + --arg url "$html_url" \ + --arg author "$author" \ + --arg assoc "$assoc" \ + --arg kind "$kind" \ + --arg action "$action" \ + --argjson labels "$labels" \ + --arg body "$body_safe" \ + --arg nudge "$nudge" \ + '{text: ( + "Event: " + $kind + "." + $action + "\n" + + "Repo: " + $repo + "\n" + + "Issue: #" + $num + " \"" + $title + "\"\n" + + "URL: " + $url + "\n" + + "Author: @" + $author + " (association: " + $assoc + ")\n" + + "Labels: " + ($labels | join(", ")) + "\n" + + (if $nudge == "" then "" else $nudge + "\n" end) + + "\n" + + "<<>>\n" + + $body + "\n" + + "<<>>" + )}') + +set +e +http_code=$(curl --fail-with-body -sS -o /tmp/triage-local-response.json -w "%{http_code}" \ + -X POST "$CLAUDE_ROUTINE_TRIAGE_URL" \ + -H "Authorization: Bearer $CLAUDE_ROUTINE_TRIAGE_TOKEN" \ + -H "anthropic-beta: experimental-cc-routine-2026-04-01" \ + -H "anthropic-version: 2023-06-01" \ + -H "Content-Type: application/json" \ + -d "$payload") +curl_rc=$? +set -e + +echo "HTTP $http_code" +sed 's/[Bb]earer [A-Za-z0-9._-]*/Bearer [REDACTED]/g' /tmp/triage-local-response.json +echo + +if [ $curl_rc -ne 0 ]; then + echo "error: curl failed (rc=$curl_rc)" >&2 + exit 1 +fi + +if [ "${http_code:-000}" -ge 400 ]; then + echo "error: routine returned HTTP $http_code" >&2 + exit 1 +fi + +echo "✓ Fired triage routine for $REPO#$ISSUE_NUM" +echo " Watch for the claude-triaging label to appear, then claude-triaged + outcome comment." diff --git a/.changeset/triage-recovery-and-local.md b/.changeset/triage-recovery-and-local.md new file mode 100644 index 0000000000..ec380ec43f --- /dev/null +++ b/.changeset/triage-recovery-and-local.md @@ -0,0 +1,3 @@ +--- +--- + diff --git a/.github/workflows/clear-stuck-claude-triaging.yml b/.github/workflows/clear-stuck-claude-triaging.yml new file mode 100644 index 0000000000..0cd4eb75f6 --- /dev/null +++ b/.github/workflows/clear-stuck-claude-triaging.yml @@ -0,0 +1,78 @@ +name: Clear stuck claude-triaging labels + +# Removes the `claude-triaging` lifecycle label from issues where it's been +# applied for >30 minutes. The label is the "I'm on this" signal for the +# triage routine; under normal flow the routine swaps it for `claude-triaged` +# at the end of the run (1-3 min). Anything older means the routine errored +# mid-run and orphaned the label. +# +# Without this sweep, orphaned `claude-triaging` labels stay forever and +# poison the "is anyone working on this?" signal for humans. + +on: + schedule: + - cron: '*/30 * * * *' # every 30 minutes + workflow_dispatch: {} + +permissions: + issues: write + contents: read + +concurrency: + group: clear-stuck-claude-triaging + cancel-in-progress: false + +jobs: + clear: + name: Clear stuck labels + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Find and clear stuck claude-triaging labels + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + # List open issues currently labeled claude-triaging. + # Filter out PRs (the label is meant for issues only). + mapfile -t numbers < <( + gh api "repos/$REPO/issues?labels=claude-triaging&state=open&per_page=100" --paginate \ + --jq '.[] | select(.pull_request == null) | .number' + ) + + if [ ${#numbers[@]} -eq 0 ]; then + echo "::notice::No issues currently labeled claude-triaging." + exit 0 + fi + + echo "Checking ${#numbers[@]} issues with claude-triaging label..." + cleared=0 + + for num in "${numbers[@]}"; do + # Find the most recent labeled-claude-triaging timeline event. + added_at=$( + gh api "repos/$REPO/issues/$num/timeline?per_page=100" --paginate \ + --jq '[.[] | select(.event == "labeled" and .label.name == "claude-triaging")] | last | .created_at // empty' + ) + + if [ -z "$added_at" ]; then + echo "::warning::#$num has claude-triaging label but no labeled event found — skipping (rare race)." + continue + fi + + now_seconds=$(date -u +%s) + added_seconds=$(date -u -d "$added_at" +%s) + age_minutes=$(( (now_seconds - added_seconds) / 60 )) + + if [ $age_minutes -gt 30 ]; then + echo "Clearing stuck claude-triaging on #$num (applied $age_minutes min ago at $added_at)" + gh issue edit "$num" --repo "$REPO" --remove-label claude-triaging + cleared=$((cleared + 1)) + else + echo " #$num — claude-triaging applied $age_minutes min ago (within 30 min window, leaving)." + fi + done + + echo "::notice::Cleared $cleared stuck claude-triaging label(s) of ${#numbers[@]} checked." diff --git a/.github/workflows/triage-webhook-miss-sweep.yml b/.github/workflows/triage-webhook-miss-sweep.yml new file mode 100644 index 0000000000..9b51bbec15 --- /dev/null +++ b/.github/workflows/triage-webhook-miss-sweep.yml @@ -0,0 +1,140 @@ +name: Triage webhook-miss sweep + +# Catches issues that the `Claude Issue Triage` workflow should have run on +# but didn't (silent webhook misses). For every open, non-bot issue created +# in the last 24h that lacks claude-triaged / claude-triaging labels AND +# has no `## Triage` comment from the routine, fire the routine manually. +# +# Motivated by issue #3112 in adcp where a normal issue creation event never +# triggered the triage workflow — webhook delivery silently dropped, no audit +# trail, the issue sat unprocessed until a human noticed. + +on: + schedule: + - cron: '17 * * * *' # hourly, offset to avoid the top of the hour + workflow_dispatch: {} + +permissions: + issues: read + contents: read + +concurrency: + group: triage-webhook-miss-sweep + cancel-in-progress: false + +jobs: + sweep: + name: Catch missed issues + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Find untriaged issues + fire routine + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + ROUTINE_URL: ${{ secrets.CLAUDE_ROUTINE_TRIAGE_URL }} + ROUTINE_TOKEN: ${{ secrets.CLAUDE_ROUTINE_TRIAGE_TOKEN }} + run: | + set -euo pipefail + + if [ -z "${ROUTINE_URL:-}" ] || [ -z "${ROUTINE_TOKEN:-}" ]; then + echo "::warning::CLAUDE_ROUTINE_TRIAGE_URL or _TOKEN not set — skipping." + exit 0 + fi + + cutoff=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) + echo "Looking for untriaged issues created since $cutoff..." + + # Open issues created in last 24h, not PRs, not bot-authored, + # not already labeled claude-triaged or claude-triaging. + mapfile -t numbers < <( + gh api "repos/$REPO/issues?state=open&since=$cutoff&per_page=100" --paginate \ + --jq '.[] | select( + .pull_request == null + and (.user.type != "Bot") + and ((.user.login | endswith("[bot]")) | not) + and (.created_at >= "'"$cutoff"'") + and ([.labels[].name] | (contains(["claude-triaged"]) or contains(["claude-triaging"])) | not) + ) | .number' + ) + + if [ ${#numbers[@]} -eq 0 ]; then + echo "::notice::No untriaged issues from last 24h." + exit 0 + fi + + echo "Found ${#numbers[@]} candidate issues without triage labels: ${numbers[*]}" + fired=0 + skipped=0 + + for num in "${numbers[@]}"; do + # Belt-and-suspenders: skip if a `## Triage` comment already + # exists. The label might have been removed manually. + has_triage_comment=$( + gh api "repos/$REPO/issues/$num/comments" --paginate \ + --jq '[.[] | select(.body | startswith("## Triage"))] | length' + ) + if [ "$has_triage_comment" -gt 0 ]; then + echo " #$num — has ## Triage comment already, skipping." + skipped=$((skipped + 1)) + continue + fi + + echo "Firing triage manually for missed issue #$num" + + issue=$(gh api "repos/$REPO/issues/$num") + title=$(echo "$issue" | jq -r '.title') + body=$(echo "$issue" | jq -r '.body // ""') + author=$(echo "$issue" | jq -r '.user.login') + assoc=$(echo "$issue" | jq -r '.author_association // "NONE"') + labels=$(echo "$issue" | jq -c '[.labels[].name]') + html_url=$(echo "$issue" | jq -r '.html_url') + + body_safe=$(printf '%s' "$body" | tr -d '\000' | head -c 8192) + + payload=$(jq -n \ + --arg repo "$REPO" \ + --arg num "$num" \ + --arg title "$title" \ + --arg url "$html_url" \ + --arg author "$author" \ + --arg assoc "$assoc" \ + --argjson labels "$labels" \ + --arg body "$body_safe" \ + '{text: ( + "Event: recovery.swept\n" + + "Repo: " + $repo + "\n" + + "Issue: #" + $num + " \"" + $title + "\"\n" + + "URL: " + $url + "\n" + + "Author: @" + $author + " (association: " + $assoc + ")\n" + + "Labels: " + ($labels | join(", ")) + "\n" + + "RECOVERY SWEEP: this issue was created >0h ago without triage labels and without a ## Triage comment. The original webhook likely missed. Treat as a fresh auto.opened event.\n" + + "\n" + + "<<>>\n" + + $body + "\n" + + "<<>>" + )}') + + set +e + http_code=$(curl --fail-with-body -sS -o /tmp/fire-response.json -w "%{http_code}" \ + -X POST "$ROUTINE_URL" \ + -H "Authorization: Bearer $ROUTINE_TOKEN" \ + -H "anthropic-beta: experimental-cc-routine-2026-04-01" \ + -H "anthropic-version: 2023-06-01" \ + -H "Content-Type: application/json" \ + -d "$payload") + curl_rc=$? + set -e + + if [ $curl_rc -ne 0 ] || [ "${http_code:-000}" -ge 400 ]; then + echo "::error::Failed to fire routine for #$num (HTTP $http_code, curl rc=$curl_rc)" + sed 's/[Bb]earer [A-Za-z0-9._-]*/Bearer [REDACTED]/g' /tmp/fire-response.json || true + continue + fi + + fired=$((fired + 1)) + # Throttle a bit so we don't fire 10 routines in 1 second. + sleep 5 + done + + echo "::notice::Fired $fired routine(s); skipped $skipped (already had ## Triage comment) of ${#numbers[@]} checked."