From 4cd379b86cf7b61ff9f9315a62de20bcf7df9f78 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 14:33:39 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#675=20?= =?UTF-8?q?=E2=80=94=20canary=20#668=20increment=202:=20SUSPECT=20triage?= =?UTF-8?q?=20=E2=80=94=20suspect=5Ffailure=5Fclasses=20+=20classify=5Ffai?= =?UTF-8?q?lure=20third=20verdict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/canary-rollout.sh | 90 +++++++++++++++---- scripts/lib/canary-rollout.sh | 20 +++-- standards/canary-rings.json | 10 +++ tests/canary_rollout.bats | 158 +++++++++++++++++++++++++++++++++- 4 files changed, 256 insertions(+), 22 deletions(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index ca76f4c85..f93be06a8 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -276,6 +276,18 @@ _benign_patterns() { | .[] | [(.workflow // ""), (.step // "")] | @tsv' } +# _suspect_patterns — emit the per-reusable SUSPECT failure-class allowlist +# (#668 increment 2) as TSV "\t", one entry per line. Empty if +# none. Unlike benign, suspect classes are emitted regardless of differs: a SUSPECT verdict +# only surfaces at differs=1 (see classify_failure), but the class always participates so a +# differs=1 counted failure is triaged SUSPECT (blocks + carries guidance) instead of a bare +# REGRESSION. Suspect classes are NEVER excluded from cum_fail — they still block. +_suspect_patterns() { + _jq -r --arg a "$1" \ + '.agents[$a]?.gate?.suspect_failure_classes? // [] + | .[] | [(.workflow // ""), (.step // "")] | @tsv' +} + # Memoization cache for _run_signature: keyed by "repo:run_id". # Avoids duplicate gh run view calls for the same (repo, run_id) across agents # in evaluate-all (where multiple agents can share repos). @@ -312,35 +324,58 @@ _failure_benign() { return 1 } +# _failure_suspect — return 0 if this +# in-window failure matches any SUSPECT allowlist entry (#668 increment 2), else 1. Reuses +# the benign_match pure matcher against the run's failed-step signature; fail-closed on an +# empty signature (an unknown signature is never treated as suspect). A suspect match does +# NOT exclude the failure — it narrows the triage verdict (SUSPECT vs REGRESSION) only. +_failure_suspect() { + local repo="$1" rid="$2" rwf="$3" patterns="$4" sig wf_re step_re + sig="$(_run_signature "$repo" "$rid")" + [ -z "$sig" ] && return 1 + while IFS=$'\t' read -r wf_re step_re; do + [ -z "$step_re" ] && continue + if [ "$(benign_match "$rwf" "$sig" "$wf_re" "$step_re")" = "yes" ]; then return 0; fi + done <<< "$patterns" + return 1 +} + # _cumulative_health — failures + # startup_failures across EVERY given tier repo since the candidate cut. Failures # matching the per-reusable known-benign allowlist (#1025 P2) are counted separately and # excluded from the blocking total; which allowlist entries apply depends on whether the -# candidate changed the reusable (differs — see _benign_patterns, #668). Prints -# " ". +# candidate changed the reusable (differs — see _benign_patterns, #668). A counted (non- +# benign) failure that matches a `suspect_failure_classes` entry sets the suspect flag +# (#668 increment 2) — it still counts toward the blocking total (SUSPECT blocks like +# REGRESSION), but downstream triage renders SUSPECT + guidance instead of a bare +# REGRESSION. Prints " ". _cumulative_health() { local agent="$1" since="$2" differs="$3"; shift 3 - local wf repo json fail=0 startup=0 benign=0 patterns="" rid rwf + local wf repo json fail=0 startup=0 benign=0 suspect=0 patterns="" suspect_patterns="" rid rwf wf="$(_agent_field "$agent" run_workflow)" patterns="$(_benign_patterns "$agent" "$differs")" + suspect_patterns="$(_suspect_patterns "$agent")" for repo in "$@"; do json="$(_run_json "$repo" "$wf" "$since")" startup=$(( startup + $(jq '[.[]?|select(.conclusion=="startup_failure")]|length' 2>/dev/null <<< "$json" || echo 0) )) - if [ -z "$patterns" ]; then - # No benign patterns to match — count all failures with one jq pass, + if [ -z "$patterns" ] && [ -z "$suspect_patterns" ]; then + # No benign or suspect patterns to match — count all failures with one jq pass, # avoiding a gh run view call per failure. fail=$(( fail + $(jq '[.[]?|select(.conclusion=="failure")]|length' 2>/dev/null <<< "$json" || echo 0) )) else while IFS=$'\t' read -r rid rwf; do - if _failure_benign "$repo" "$rid" "$rwf" "$patterns"; then + if [ -n "$patterns" ] && _failure_benign "$repo" "$rid" "$rwf" "$patterns"; then benign=$(( benign + 1 )) else fail=$(( fail + 1 )) + if [ -n "$suspect_patterns" ] && _failure_suspect "$repo" "$rid" "$rwf" "$suspect_patterns"; then + suspect=1 + fi fi done < <(jq -r '.[]?|select(.conclusion=="failure")|[(.databaseId // "" | tostring),(.workflowName // "")]|@tsv' 2>/dev/null <<< "$json") fi done - echo "$fail $startup $benign" + echo "$fail $startup $benign $suspect" } # _baseline_daily — per-day EXECUTED counts on the @@ -452,8 +487,8 @@ _frontier_state() { while IFS= read -r r; do [ -n "$r" ] && [ "$r" != '*' ] && all_repos+=("$r"); done \ < <(resolve_members "$agent" "$ch3") done - local cum_fail cum_startup cum_benign - read -r cum_fail cum_startup cum_benign < <(_cumulative_health "$agent" "$cut_z" "$differs" "${all_repos[@]}") + local cum_fail cum_startup cum_benign cum_suspect + read -r cum_fail cum_startup cum_benign cum_suspect < <(_cumulative_health "$agent" "$cut_z" "$differs" "${all_repos[@]}") # Per-transition knobs (registry-configurable; #548 defaults live in the ring SoT). local dwell_floor waived="false" target=0 @@ -483,7 +518,7 @@ _frontier_state() { local triage="-" if [ "$state" = "BLOCKED" ]; then - triage="$(classify_failure "$differs" "${CANARY_FAILURE_CATEGORY:-unknown}")" + triage="$(classify_failure "$differs" "${CANARY_FAILURE_CATEGORY:-unknown}" "$cum_suspect")" fi echo "$cand $frontier $transition $state $dwell_h $dwell_floor $sample $target $cum_fail $cum_startup $cum_benign $triage" @@ -512,6 +547,8 @@ cmd_evaluate() { if [ "$state" = "BLOCKED" ]; then if [ "$triage" = "REGRESSION" ]; then echo "::error::triage=REGRESSION — candidate changed the reusable and a run failed since cut. HALT + hold; recommend rollback." + elif [ "$triage" = "SUSPECT" ]; then + echo "::warning::triage=SUSPECT — failure matches a suspect class (possibly candidate-caused). BLOCKS + needs a human; see the blocker issue's discriminating question, then promote --override if unrelated or roll back if a real regression." else echo "::warning::triage=PRE_EXISTING — failure is pre-existing/environmental. Report only; do NOT rollback, do NOT advance." fi @@ -558,8 +595,13 @@ cmd_promote() { local allow_pre allow_pre="$(_jq -r --arg a "$agent" '.agents[$a].gate?.control?.allow_pre_existing // false')" [ "$allow_pre_flag" = true ] && allow_pre=true - if [ "$state" = "BLOCKED" ] && [ "$triage" = "REGRESSION" ] && [ "$override" != true ]; then - echo "::error::gate=BLOCKED (triage=REGRESSION) for '$frontier' [$transition] — candidate regression suspected; not promoting. Investigate + rollback, do not --override blindly." + # REGRESSION and SUSPECT both HALT + need a human: neither advances without --override, + # and --allow-pre-existing (which only unblocks PRE_EXISTING) never advances them. For a + # SUSPECT the human answers the class's discriminating question first — `--override` when + # the timeout is unrelated to the diff, or roll back when the candidate got materially + # slower (#668 increment 2). + if [ "$state" = "BLOCKED" ] && { [ "$triage" = "REGRESSION" ] || [ "$triage" = "SUSPECT" ]; } && [ "$override" != true ]; then + echo "::error::gate=BLOCKED (triage=$triage) for '$frontier' [$transition] — candidate regression suspected; not promoting. Investigate + rollback, do not --override blindly." return 0 fi local advance=false @@ -716,11 +758,28 @@ _blocker_evidence() { printf '%s' "$out" } +# _suspect_guidance — emit the discriminating question(s) for the agent's +# suspect_failure_classes (#668 increment 2), one bullet per class, so a SUSPECT blocker +# tells the human exactly how to confirm in ~30 seconds. Empty if the agent has none. +_suspect_guidance() { + _jq -r --arg a "$1" \ + '.agents[$a]?.gate?.suspect_failure_classes? // [] + | .[] | select((.guidance // "") != "") + | "- **\(.id):** \(.guidance)"' +} + # _blocker_body _blocker_body() { local agent="$1" transition="$2" cand="$3" cum_fail="$4" cum_startup="$5" triage="$6" host="$7" evidence="$8" note if [ "$triage" = "REGRESSION" ]; then note="> ⛔ **REGRESSION** — the candidate changed the reusable and a run failed since its cut. HALT + hold; investigate and roll back rather than \`--override\`. (labelled \`needs-human\`)" + elif [ "$triage" = "SUSPECT" ]; then + local guidance; guidance="$(_suspect_guidance "$agent")" + [ -z "$guidance" ] && guidance="- _(no per-class guidance registered)_" + note="> ⚠️ **SUSPECT** — the candidate changed the reusable and a run failed with a *possibly-candidate-caused* signature. This still BLOCKS and needs a human (labelled \`needs-human\`), but answer the discriminating question below to confirm fast: if unrelated to the diff, \`promote --override\`; if the candidate is materially responsible, treat it as a real regression and roll back. +> +> **Discriminating question:** +$(printf '%s\n' "$guidance" | sed 's/^/> /')" else note="> ⚠️ **PRE_EXISTING** — the failure is pre-existing/environmental (reusable byte-identical to the prior channel). Report only; the gate will not roll back or advance. Fix-forward, and the armed timer auto-promotes once clean." fi @@ -779,8 +838,9 @@ cmd_sync_issues() { # Route blockers so they get ACTIONED, not left sitting: `dev-lead` (dev-lead-intent # treats a `dev-lead`-labelled issue as an actionable "issue" intent and picks it up — # the App token applies the label, so the `issues: labeled` event DOES trigger the agent, - # unlike a github.token edit). `needs-human` additionally flags REGRESSION blockers, which - # the gate escalates to a human (roll back rather than blind --override). + # unlike a github.token edit). `needs-human` additionally flags REGRESSION and SUSPECT + # blockers, which the gate escalates to a human (roll back, or confirm the discriminating + # question then --override, rather than a blind --override). gh label create dev-lead --repo "$ISSUE_REPO" --color 5319e7 --description "Route to the dev-lead agent for action" >/dev/null 2>&1 || true gh label create needs-human --repo "$ISSUE_REPO" --color d93f0b --description "Requires human judgement (canary regression)" >/dev/null 2>&1 || true fi @@ -807,7 +867,7 @@ cmd_sync_issues() { num="$(_gh_issue_create "$title" "$body" "canary-blocker" || true)" if [ -n "$num" ]; then gh issue edit "$num" --repo "$ISSUE_REPO" --add-label dev-lead >/dev/null 2>&1 || true - [ "$triage" = "REGRESSION" ] && gh issue edit "$num" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true + { [ "$triage" = "REGRESSION" ] || [ "$triage" = "SUSPECT" ]; } && gh issue edit "$num" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true echo " opened blocker issue #$num for $agent"; blk="#$num" else echo "::warning::could not open blocker issue for $agent (Issues:write on the App?)"; fi fi diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh index fc7cb5bfa..4729f2ddd 100644 --- a/scripts/lib/canary-rollout.sh +++ b/scripts/lib/canary-rollout.sh @@ -158,19 +158,29 @@ decide_graduated() { echo "SOAKING" } -# classify_failure — triage an in-window failure -# (called only when decide_graduated returns BLOCKED). Echoes: +# classify_failure [suspect_match 0|1] — triage an +# in-window failure (called only when decide_graduated returns BLOCKED). Echoes: # REGRESSION — the candidate changed the reusable (differs=1) AND the failure is -# not a known environmental class → HALT, auto-hold, recommend rollback. +# not a known environmental class and not a suspect class → HALT, +# auto-hold, recommend rollback. +# SUSPECT — the candidate changed the reusable (differs=1) AND the failure matches +# a `suspect_failure_classes` entry (#668 increment 2): a possibly- +# candidate-caused class (e.g. an exit-124 workload timeout) that COULD be +# a real regression, so it still BLOCKS + needs a human — but carries a +# per-class discriminating question so the human confirm is fast. Unlike a +# `version_independent` benign class it is NOT auto-cleared. # PRE_EXISTING — the reusable is byte-identical to the prior channel (differs=0), OR # the failure is environmental (comment-cap / rate-limit / infra / data) # → report, do NOT rollback, do NOT advance. +# Precedence: environmental category first, then differs=0 (can't be candidate-caused), +# then a suspect match narrows an otherwise-REGRESSION verdict to SUSPECT. classify_failure() { - local differs="${1:-0}" category="${2:-unknown}" + local differs="${1:-0}" category="${2:-unknown}" suspect="${3:-0}" case "$category" in comment-cap|rate-limit|infra|data) echo "PRE_EXISTING"; return 0 ;; esac - if [ "$differs" = "1" ]; then echo "REGRESSION"; else echo "PRE_EXISTING"; fi + if [ "$differs" != "1" ]; then echo "PRE_EXISTING"; return 0; fi + if [ "$suspect" = "1" ]; then echo "SUSPECT"; else echo "REGRESSION"; fi } # benign_match diff --git a/standards/canary-rings.json b/standards/canary-rings.json index 80db93ef7..05cd22611 100644 --- a/standards/canary-rings.json +++ b/standards/canary-rings.json @@ -70,6 +70,16 @@ "allow_pre_existing": false }, "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. Exception (#668): a class marked `version_independent: true` fails before/independent of the candidate's own code (e.g. a Dependabot-context secrets failure at startup) and can NEVER be candidate-caused, so it is excluded even when differs=1 \u2014 reserve the flag for classes where that is provable from the failure mechanism, not merely likely. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "suspect_failure_classes": [ + { + "id": "workload-timeout", + "workflow": "Dev-Lead Agent", + "step": "Stage timeout|exit code 124|rebase-conflict-sentinel", + "reason": "#664/#668 \u2014 a dev-lead run can exit 124 when a stage exceeds its time budget (large workload, or a rebase-conflict sentinel that never resolves). This is *possibly* candidate-caused (a candidate that changed execution-time-sensitive code could genuinely be slower), so unlike a version_independent benign class it is NOT auto-cleared \u2014 it BLOCKS as SUSPECT and carries a discriminating question for a fast human confirm.", + "guidance": "Did this candidate change execution-time-sensitive code, or is it a large-workload/rebase timeout unrelated to the diff? If unrelated \u2192 `promote --override`; if the candidate got materially slower \u2192 treat as a real regression and roll back." + } + ], + "_suspect_note": "suspect_failure_classes (#668 increment 2): same shape as benign classes (`id`, `workflow` ERE, `step` ERE) plus a `guidance` discriminating question. A counted (non-benign) failure that matches a suspect class at differs=1 is triaged SUSPECT instead of REGRESSION \u2014 it still BLOCKS and still needs a human (labelled `needs-human`, NOT auto-cleared like `version_independent`), but the blocker issue renders the `guidance` so the human confirm is a 30-second check rather than a from-scratch investigation. Default-absent (this key omitted) = today's behavior, byte-identical for agents that do not opt in.", "transitions": { "next->ring0": { "dwell_hours": 4, diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index f147470d6..c0211d507 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -134,8 +134,8 @@ setup() { [ "$(decide_graduated 5 8 0 0 true 0 0)" = "SOAKING" ] } -# ── classify_failure (triage: regression vs pre-existing/environmental) ──────── -# args: +# ── classify_failure (triage: regression vs pre-existing/environmental/suspect) ─ +# args: [suspect_match 0|1] @test "classify_failure: reusable changed + non-environmental → REGRESSION" { [ "$(classify_failure 1 unknown)" = "REGRESSION" ] } @@ -149,6 +149,25 @@ setup() { [ "$(classify_failure 1 data)" = "PRE_EXISTING" ] } +# ── classify_failure: SUSPECT third verdict (#668 increment 2, #675) ──────────── +# A suspect-class match at differs=1 becomes SUSPECT instead of REGRESSION — it still +# BLOCKS + needs a human, but carries a discriminating question so the confirm is fast. +@test "classify_failure: suspect match + reusable changed → SUSPECT (not REGRESSION)" { + [ "$(classify_failure 1 unknown 1)" = "SUSPECT" ] +} +@test "classify_failure: NO suspect match + reusable changed → still REGRESSION" { + [ "$(classify_failure 1 unknown 0)" = "REGRESSION" ] +} +@test "classify_failure: suspect match but reusable identical → PRE_EXISTING (can't be candidate-caused)" { + [ "$(classify_failure 0 unknown 1)" = "PRE_EXISTING" ] +} +@test "classify_failure: environmental category beats a suspect match → PRE_EXISTING" { + [ "$(classify_failure 1 infra 1)" = "PRE_EXISTING" ] +} +@test "classify_failure: suspect arg defaults to 0 (omitted) → REGRESSION at differs=1" { + [ "$(classify_failure 1 unknown)" = "REGRESSION" ] +} + # ── _reusable_differs: cross-repo host-aware blob compare (#613) ─────────────── # When an agent's host != THIS_REPO (e.g. dev-lead hosted in .github-private while the # engine runs from .github), the reusable blob is NOT in the local checkout — the compare @@ -1353,3 +1372,138 @@ GITEOF run jq -e '.agents["ci-failure-analyst"].gate.benign_failure_classes == []' "$RINGS" [ "$status" -eq 0 ] } + +# ── SUSPECT triage: suspect_failure_classes (#668 increment 2, #675) ───────────── +# A *possibly-candidate-caused* failure class (dev-lead exit-124 workload timeouts) gets +# the full REGRESSION verdict at differs=1 today, forcing ad-hoc human diagnosis each time. +# A `suspect_failure_classes` entry (matched even at differs=1, unlike benign) instead +# yields SUSPECT: still BLOCKS + needs a human, but carries a discriminating question so the +# confirm is a 30-second check. Default-absent = today's behaviour for agents without it. + +@test "_suspect_patterns: emits the dev-lead workload-timeout class (wf/step TSV)" { + run env CANARY_RINGS="$RINGS" bash -c "source '$ORCH' && _suspect_patterns dev-lead" + [ "$status" -eq 0 ] + [ "$(wc -l <<< "$output")" -eq 1 ] + [[ "$output" == *"Dev-Lead Agent"* ]] + [[ "$output" == *"Stage timeout"* ]] +} + +@test "_suspect_patterns: unknown agent key → empty output, no crash (null-safety)" { + run env CANARY_RINGS="$RINGS" bash -c "source '$ORCH' && _suspect_patterns __nonexistent_agent__" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "_suspect_patterns: agent with no gate field → empty output, no crash (null-safety)" { + local tmp_rings + tmp_rings="$(mktemp "$BATS_TEST_TMPDIR/rings-nogate.XXXXXX.json")" + jq '.agents["no-gate-agent"] = {}' "$RINGS" > "$tmp_rings" + run env CANARY_RINGS="$tmp_rings" bash -c "source '$ORCH' && _suspect_patterns no-gate-agent" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +@test "_suspect_patterns: agent without suspect_failure_classes → empty (default-off, byte-identical)" { + # agent-shield opts out entirely (no suspect_failure_classes key) → today's behaviour. + run env CANARY_RINGS="$RINGS" bash -c "source '$ORCH' && _suspect_patterns agent-shield" + [ "$status" -eq 0 ] + [ -z "$output" ] +} + +# _suspect_stub — like _vi_benign_stub: dev-lead is cross-repo +# (host=.github-private), reusable DIFFERS (reuseAAAA vs reuseBBBB → _reusable_differs=1), +# every tier repo returns failure runs whose failed step is . Feeds the +# suspect-class check at differs=1 (where the benign allowlist would otherwise disable). +_suspect_stub() { + local failed_step="$1" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + local cut_iso run_iso + cut_iso="$(date -u -d "-3 days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-3d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d "-2 days" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-2d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/gh" < "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # no local refs for a cross-repo agent +GITEOF + chmod +x "$STUB_BIN/git" +} + +@test "orchestrator: differs=1 failure matching the suspect class → BLOCKED + SUSPECT (not REGRESSION)" { + # The exit-124 workload-timeout signature is a suspect class → even though the candidate + # changed the reusable, it triages SUSPECT (still blocks + needs a human), not REGRESSION. + _suspect_stub "Stage timeout (exit code 124)" + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"SUSPECT"* ]] + [[ "$output" != *"REGRESSION"* ]] +} + +@test "orchestrator: differs=1 non-suspect failure → still BLOCKED + REGRESSION" { + # A failure that matches no suspect class stays a full REGRESSION at differs=1. + _suspect_stub "Compile TypeScript" + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"REGRESSION"* ]] + [[ "$output" != *"SUSPECT"* ]] +} + +@test "orchestrator: SUSPECT still BLOCKS — promote refuses without --override" { + _suspect_stub "Stage timeout (exit code 124)" + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"SUSPECT"* ]] + [[ "$output" != *"DRY-RUN"* ]] # no move planned — the gate held +} + +@test "orchestrator: SUSPECT is NOT advanced by --allow-pre-existing (only --override)" { + _suspect_stub "Stage timeout (exit code 124)" + run env CANARY_RINGS="$RINGS" bash "$ORCH" promote dev-lead --allow-pre-existing --dry-run + [ "$status" -eq 0 ] + [[ "$output" == *"SUSPECT"* ]] + [[ "$output" != *"DRY-RUN"* ]] +} + +@test "canary-rings.json: dev-lead gate carries a suspect_failure_classes allowlist with guidance" { + run jq -e '.agents["dev-lead"].gate.suspect_failure_classes | type == "array" and length >= 1' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.suspect_failure_classes + | all(has("id") and has("workflow") and has("step") and has("reason") and has("guidance"))' "$RINGS" + [ "$status" -eq 0 ] + run jq -e '.agents["dev-lead"].gate.suspect_failure_classes[] + | select(.id=="workload-timeout") | .step | test("124")' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "canary-rings.json: suspect_failure_classes is default-absent for opted-out agents (byte-identical)" { + run jq -e '.agents["agent-shield"].gate | has("suspect_failure_classes") | not' "$RINGS" + [ "$status" -eq 0 ] +} + +@test "_blocker_body: SUSPECT triage renders the class guidance (discriminating question)" { + # The blocker body must surface the workload-timeout discriminating question prominently + # so a human confirm is a fast check, and keep the SUSPECT label + needs-human routing. + run env CANARY_RINGS="$RINGS" bash -c \ + "source '$ORCH' && _blocker_body dev-lead 'next->ring0' cccccccccccc 1 0 SUSPECT petry-projects/.github-private '_(evidence)_'" + [ "$status" -eq 0 ] + [[ "$output" == *"SUSPECT"* ]] + [[ "$output" == *"needs-human"* ]] + [[ "$output" == *"override"* ]] # the guidance names the fast path when unrelated +} From 7abb0a97b5e0db91d0b2e9ef377012462294bdc4 Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 14:38:37 +0000 Subject: [PATCH 2/3] fix(reviews): address review comments [skip ci-relay] --- scripts/canary-rollout.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index f93be06a8..9fea39ba3 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -333,7 +333,9 @@ _failure_suspect() { local repo="$1" rid="$2" rwf="$3" patterns="$4" sig wf_re step_re sig="$(_run_signature "$repo" "$rid")" [ -z "$sig" ] && return 1 - while IFS=$'\t' read -r wf_re step_re; do + while IFS=$'\t' read -r wf_re step_re || [ -n "$wf_re" ]; do + wf_re="${wf_re%$'\r'}" + step_re="${step_re%$'\r'}" [ -z "$step_re" ] && continue if [ "$(benign_match "$rwf" "$sig" "$wf_re" "$step_re")" = "yes" ]; then return 0; fi done <<< "$patterns" From f0e5f1b7ad6ff5846fa75a9ce2623384dd42326d Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 14:49:03 +0000 Subject: [PATCH 3/3] chore: dev-lead update (review-changes) [skip ci-relay] --- scripts/canary-rollout.sh | 2 +- tests/canary_rollout.bats | 65 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 9fea39ba3..1dc3a8b3f 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -879,7 +879,7 @@ cmd_sync_issues() { gh issue edit "$num" --repo "$ISSUE_REPO" --title "$title" --body "$body" >/dev/null 2>&1 \ || echo "::warning::could not update blocker issue #$num for $agent" gh issue edit "$num" --repo "$ISSUE_REPO" --add-label dev-lead >/dev/null 2>&1 || true - [ "$triage" = "REGRESSION" ] && gh issue edit "$num" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true + { [ "$triage" = "REGRESSION" ] || [ "$triage" = "SUSPECT" ]; } && gh issue edit "$num" --repo "$ISSUE_REPO" --add-label needs-human >/dev/null 2>&1 || true echo " updated blocker issue #$num for $agent"; blk="#$num" fi fi diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index c0211d507..43a374b4a 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1507,3 +1507,68 @@ GITEOF [[ "$output" == *"needs-human"* ]] [[ "$output" == *"override"* ]] # the guidance names the fast path when unrelated } + +# ── sync-issues needs-human label routing for SUSPECT triage ───────────────────────── +# The create path applies needs-human for both REGRESSION and SUSPECT (fixed in 4cd379b). +# The update path must match: when an existing open blocker is refreshed with SUSPECT +# triage, needs-human must be re-applied so escalation routing is not lost on re-runs. +# +# _sync_suspect_stub — like _sync_stub but with differs=1 blobs (reuseAAAA vs reuseBBBB) +# and failure runs whose failed step matches the dev-lead workload-timeout suspect class. +_sync_suspect_stub() { + local blocker_list="${1:-[]}" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + export ISSUE_LOG="$STUB_BIN/issue.log"; : > "$ISSUE_LOG" + local cut_iso run_iso + cut_iso="$(date -u -d '-3 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-3d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + run_iso="$(date -u -d '-2 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-2d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # dev-lead is cross-repo; all tag/blob resolution goes via gh api +GITEOF + chmod +x "$STUB_BIN/git" + cat > "$STUB_BIN/gh" <> "$ISSUE_LOG"; echo "https://github.com/petry-projects/.github-private/issues/777" ;; + "issue edit"*) echo "EDIT|\$*" >> "$ISSUE_LOG" ;; + "issue close"*) echo "CLOSE|\$*" >> "$ISSUE_LOG" ;; + "issue reopen"*) echo "REOPEN|\$*" >> "$ISSUE_LOG" ;; + "label create"*) : ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + SYNC_RINGS="$BATS_TEST_TMPDIR/sync-rings-suspect.json" + jq '{org_infra_repos, agents: {"dev-lead": .agents["dev-lead"]}}' "$RINGS" > "$SYNC_RINGS" +} + +@test "orchestrator: sync-issues CREATE path applies needs-human for SUSPECT triage" { + # No existing issue → create path; SUSPECT (differs=1 + suspect step) must add needs-human. + _sync_suspect_stub '[]' + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"opened blocker issue #777 for dev-lead"* ]] + grep -q -- "--add-label needs-human" "$ISSUE_LOG" +} + +@test "orchestrator: sync-issues UPDATE path applies needs-human for SUSPECT triage" { + # Existing open blocker #501 → update path; SUSPECT triage must still add needs-human. + # The create path was fixed in 4cd379b; this test pins the update-path parity contract. + _sync_suspect_stub '[{"number":501,"state":"OPEN","body":""}]' + run env CANARY_RINGS="$SYNC_RINGS" ISSUE_REPO="petry-projects/.github-private" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"updated blocker issue #501 for dev-lead"* ]] + grep -q -- "--add-label needs-human" "$ISSUE_LOG" +}