From 7ba0d4185b54f47b370f424f6e398365170ff26b Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:01:24 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20implement=20issue=20#683=20?= =?UTF-8?q?=E2=80=94=20canary=20#668=20increment=204:=20decision=20telemet?= =?UTF-8?q?ry=20(Layer=202)=20=E2=80=94=20decision-step=20emission=20+=20d?= =?UTF-8?q?ecide=5Fdecision=5Fshift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pr-auto-review-reusable.yml | 31 +++ node_modules/.package-lock.json | 2 +- package-lock.json | 2 +- scripts/canary-rollout.sh | 215 ++++++++++++++-- scripts/lib/canary-rollout.sh | 74 ++++++ standards/canary-rings.json | 7 + tests/canary_rollout.bats | 237 ++++++++++++++++++ 7 files changed, 551 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr-auto-review-reusable.yml b/.github/workflows/pr-auto-review-reusable.yml index 3a129d110..3f904fe87 100644 --- a/.github/workflows/pr-auto-review-reusable.yml +++ b/.github/workflows/pr-auto-review-reusable.yml @@ -77,6 +77,7 @@ jobs: if [ "${{ github.event.pull_request.draft }}" = "true" ]; then echo "PR is a draft — skipping" echo "skip=true" >> "$GITHUB_OUTPUT" + echo "decision=skip-draft" >> "$GITHUB_OUTPUT" exit 0 fi PR_URL="${{ github.event.pull_request.html_url }}" @@ -147,6 +148,7 @@ jobs: if [ "$STATE" != "OPEN" ] || [ "$IS_DRAFT" = "true" ]; then echo "PR is $STATE (draft=$IS_DRAFT) — skipping" echo "ready=false" >> "$GITHUB_OUTPUT" + echo "decision=skip-draft" >> "$GITHUB_OUTPUT" exit 0 fi echo "PR is open and not a draft ✓" @@ -163,6 +165,7 @@ jobs: if [ "$TOTAL" -eq 0 ]; then echo "No CI checks found on this PR — skipping" echo "ready=false" >> "$GITHUB_OUTPUT" + echo "decision=skip-checks-pending" >> "$GITHUB_OUTPUT" exit 0 fi @@ -192,6 +195,7 @@ jobs: if ! REASON=$(echo "$CHECKS" | pr_auto_review_checks_ready "$REQUIRED_JSON" "$SELF_CHECK"); then echo "$REASON" echo "ready=false" >> "$GITHUB_OUTPUT" + echo "decision=skip-checks-pending" >> "$GITHUB_OUTPUT" exit 0 fi echo "$REASON ✓" @@ -203,6 +207,7 @@ jobs: if [ "$REVIEW_DECISION" = "CHANGES_REQUESTED" ]; then echo "Effective review decision is CHANGES_REQUESTED — skipping" echo "ready=false" >> "$GITHUB_OUTPUT" + echo "decision=skip-changes-requested" >> "$GITHUB_OUTPUT" exit 0 fi echo "No CHANGES_REQUESTED review decision ✓" @@ -224,12 +229,14 @@ jobs: if [ "$UNRESOLVED" -gt 0 ]; then echo "$UNRESOLVED unresolved review thread(s) — skipping" echo "ready=false" >> "$GITHUB_OUTPUT" + echo "decision=skip-unresolved-threads" >> "$GITHUB_OUTPUT" exit 0 fi echo "No unresolved review threads ✓" echo "All readiness criteria met — dispatching review agent" echo "ready=true" >> "$GITHUB_OUTPUT" + echo "decision=dispatched" >> "$GITHUB_OUTPUT" - name: Dispatch review agent if: steps.criteria.outputs.ready == 'true' @@ -244,3 +251,27 @@ jobs: --field event_type=pr-review-mention \ --field "client_payload[pr_url]=$PR_URL" echo "::notice::Auto-review dispatched for $PR_URL" + + # Decision telemetry (#668 increment 4, Layer 2). Each outcome branch is a no-op + # step whose NAME encodes the decision the gate reached; exactly one condition is + # true per run, so the taken branch runs (conclusion=success) and the rest are + # `skipped`. The canary-rollout engine reads these names off `gh run view --json + # jobs` (decision_class / _sample_decision_counts) to tally the decision-mix and + # detect a distribution shift in a candidate. Additive + side-effect-free: a run + # that short-circuits before a decision (n/a event, no PR) emits none, and no + # consumer other than the opt-in gate.correctness block reads them. + - name: 'decision: dispatched' + if: steps.criteria.outputs.decision == 'dispatched' + run: 'true' + - name: 'decision: skip-draft' + if: steps.pr.outputs.decision == 'skip-draft' || steps.criteria.outputs.decision == 'skip-draft' + run: 'true' + - name: 'decision: skip-checks-pending' + if: steps.criteria.outputs.decision == 'skip-checks-pending' + run: 'true' + - name: 'decision: skip-changes-requested' + if: steps.criteria.outputs.decision == 'skip-changes-requested' + run: 'true' + - name: 'decision: skip-unresolved-threads' + if: steps.criteria.outputs.decision == 'skip-unresolved-threads' + run: 'true' diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index e815605ad..5e396f0fc 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -1,5 +1,5 @@ { - "name": "pr-681", + "name": ".github", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/package-lock.json b/package-lock.json index 060eb1523..6f0776ff0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "pr-681", + "name": ".github", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 1e3bf293e..48095e169 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -427,10 +427,141 @@ _reusable_differs() { echo 0 } +# ── decision telemetry sampling (#668 Layer 2, increment 4) ───────────────────── +# Feeds the pure decision-shift core (decide_decision_shift / decision_class in the lib) +# with real decision-mix tallies gathered from `gh run view --json jobs` step names — the +# same surface the benign/suspect check already fetches. Reserved for agents that opt into +# gate.correctness; every other agent skips this path entirely (byte-identical). + +# Memoization for _run_decision_class, keyed "repo:run_id" (the decision class is +# prefix-independent given the run, so the prefix is not part of the key). Distinct from +# _RUN_SIG_CACHE: decision sampling hits SUCCESS runs, the signature cache hits FAILED runs, +# so the two never fetch the same run twice. +declare -A _RUN_DECISION_CACHE=() + +# _run_decision_class — the decision class a run took (the taken +# `` no-op step; skipped branches ignored), via `gh run view --json jobs`. +# Empty on missing repo/id or any gh error (fail-open: a run with no decision step simply +# contributes nothing to the tally, degrading toward INSUFFICIENT, never a false SHIFT). +_run_decision_class() { + local repo="$1" id="$2" prefix="$3" cache_key json cls + { [ -z "$repo" ] || [ "$repo" = '*' ] || [ -z "$id" ]; } && { echo ""; return 0; } + cache_key="${repo}:${id}" + if [[ -v _RUN_DECISION_CACHE["$cache_key"] ]]; then + echo "${_RUN_DECISION_CACHE[$cache_key]}"; return 0 + fi + json="$(gh run view "$id" --repo "$repo" --json jobs 2>/dev/null || echo '{}')" + cls="$(decision_class "$prefix" "$json")" + _RUN_DECISION_CACHE["$cache_key"]="$cls" + echo "$cls" +} + +# _sample_decision_counts — tally the +# decision-class mix of up to EXECUTED runs (newest first) on the given tier repos in +# the window [since_z, before_z). before_z="-" means no upper bound (all runs since ). +# Runs with no decision step are skipped (they do not consume a sample slot). Echoes a compact +# JSON object "":; "{}" when nothing qualifies. Pure-core (decide_decision_shift) +# reads the object; the min-sample knobs gate sufficiency. +_sample_decision_counts() { + local agent="$1" prefix="$2" max_k="$3" since="$4" before="$5"; shift 5 + local wf repo json rid cls sampled=0 + wf="$(_agent_field "$agent" run_workflow)" + declare -A counts=() + local filter='.[]?|select(.conclusion=="success" or .conclusion=="failure")' + [ "$before" != "-" ] && filter="$filter|select(.createdAt < \"$before\")" + for repo in "$@"; do + [ "$sampled" -ge "$max_k" ] && break + { [ -z "$repo" ] || [ "$repo" = '*' ]; } && continue + json="$(_run_json "$repo" "$wf" "$since")" + while IFS= read -r rid; do + [ -z "$rid" ] && continue + [ "$sampled" -ge "$max_k" ] && break + cls="$(_run_decision_class "$repo" "$rid" "$prefix")" + [ -z "$cls" ] && continue + counts["$cls"]=$(( ${counts["$cls"]:-0} + 1 )) + sampled=$(( sampled + 1 )) + done < <(jq -r "[${filter}]|sort_by(.createdAt)|reverse|.[]|(.databaseId|tostring)" 2>/dev/null <<< "$json") + done + local out='{}' k + for k in "${!counts[@]}"; do + out="$(jq -c --arg k "$k" --argjson v "${counts[$k]}" '. + {($k):$v}' <<< "$out")" + done + echo "$out" +} + +# _correctness_verdict — OK|SHIFT|INSUFFICIENT for +# an agent that opts into gate.correctness (empty when it does not). Candidate mix = source-tier +# runs since the candidate cut; baseline mix = source-tier runs in the trailing window BEFORE the +# cut (the prior version on identical traffic — the enumerable stand-in for "the incumbent on +# comparable traffic", since the stable ring is often the unenumerable `*`). Delegates the gate +# to the pure decide_decision_shift. +_correctness_verdict() { + local agent="$1" cand="$2" cut_z="$3" src_csv="$4" + local knobs prefix + knobs="$(_jq -c --arg a "$agent" '.agents[$a].gate.correctness')" + prefix="$(jq -r '.decision_step_prefix // "decision: "' <<< "$knobs" 2>/dev/null || echo "decision: ")" + local src_repos=() r + IFS=, read -r -a src_repos <<< "$src_csv" + local win base_since cand_counts base_counts + win="${SOAK_WINDOW_DAYS:-$(_gate_field "$agent" baseline_window_days)}"; win="${win:-14}" + base_since="$(_iso_now_minus_days "$win")" + # K=15 candidate / K=25 baseline — n in the 10–25 band the design targets; the min-sample + # knobs (below the caps) gate sufficiency, so a thin sample degrades to INSUFFICIENT. + cand_counts="$(_sample_decision_counts "$agent" "$prefix" 15 "$cut_z" "-" "${src_repos[@]}")" + base_counts="$(_sample_decision_counts "$agent" "$prefix" 25 "$base_since" "$cut_z" "${src_repos[@]}")" + decide_decision_shift "$cand_counts" "$base_counts" "$knobs" +} + +# _decision_mix_table — a markdown table of the candidate vs prior-version +# baseline decision-class share (permille, rounded like decide_decision_shift), for the blocker +# body of a correctness-SHIFT hold (#668 L2). Recomputes the same source-tier samples as +# _correctness_verdict (the per-run classes are memoized in _RUN_DECISION_CACHE, so the second +# pass is cache-warm). Empty when the agent has no gate.correctness or the frontier is resolved. +_decision_mix_table() { + local agent="$1" cand="$2" + local knobs; knobs="$(_jq -c --arg a "$agent" '.agents[$a].gate.correctness // ""')" + [ -z "$knobs" ] || [ "$knobs" = '""' ] && return 0 + local chans frontier="" ch c + chans="$(ordered_channels "$agent")" + local chan_array=(); IFS=, read -r -a chan_array <<< "$chans" + for ch in "${chan_array[@]}"; do + c="$(channel_commit "$agent" "$ch")" + if [ "$ch" = "next" ] || [ "$c" = "$cand" ]; then :; else frontier="$ch"; break; fi + done + [ -z "$frontier" ] && return 0 + local transition source cut_z prefix + transition="$(transition_key "$frontier" "$chans")" + source="${transition%%->*}" + cut_z="$(candidate_cut_date "$agent" "$cand")" + [ -z "$cut_z" ] && return 0 + prefix="$(jq -r '.decision_step_prefix // "decision: "' <<< "$knobs" 2>/dev/null || echo "decision: ")" + local src_repos=() r + while IFS= read -r r; do [ -n "$r" ] && src_repos+=("$r"); done < <(resolve_members "$agent" "$source") + local win base_since cand_counts base_counts + win="${SOAK_WINDOW_DAYS:-$(_gate_field "$agent" baseline_window_days)}"; win="${win:-14}" + base_since="$(_iso_now_minus_days "$win")" + cand_counts="$(_sample_decision_counts "$agent" "$prefix" 15 "$cut_z" "-" "${src_repos[@]}")" + base_counts="$(_sample_decision_counts "$agent" "$prefix" 25 "$base_since" "$cut_z" "${src_repos[@]}")" + jq -nr --argjson c "$cand_counts" --argjson b "$base_counts" --arg win "$win" ' + ([$c[]?]|add // 0) as $ct | ([$b[]?]|add // 0) as $bt + | (($c|keys) + ($b|keys) | unique) as $classes + | "| decision class | candidate | baseline | Δ |", + "|---|---|---|---|", + ( $classes[] + | ( if $ct>0 then ((( ($c[.]//0)*1000 + ($ct/2) ) / $ct) | floor) else 0 end ) as $cs + | ( if $bt>0 then ((( ($b[.]//0)*1000 + ($bt/2) ) / $bt) | floor) else 0 end ) as $bs + | (($cs - $bs) | if . < 0 then -. else . end) as $d + | "| `\(.)` | \($cs/10)% (\($c[.]//0)) | \($bs/10)% (\($b[.]//0)) | \($d/10)pp |" ), + "", + "_candidate n=\($ct) (source tier since cut) · baseline n=\($bt) (prior version, trailing \($win) d). Share = rounded per-mille; Δ is the absolute shift the gate compares to `max_shift_permille`._" + ' 2>/dev/null || return 0 +} + # _frontier_state — compute the rollout frontier and graduated gate, echoing: -# " " +# " " # frontier = first ring (after next) not yet on the candidate commit; triage is "-" -# unless state is BLOCKED (then REGRESSION | PRE_EXISTING). +# unless state is BLOCKED (then REGRESSION | PRE_EXISTING | SUSPECT). mix_shift is "SHIFT" +# when a gate.correctness decision-mix shift is holding the promotion (#668 L2), else "-". _frontier_state() { local agent="$1" local cand chans frontier="" @@ -445,7 +576,7 @@ _frontier_state() { if [ "$ch" = "next" ] || [ "$c" = "$cand" ]; then :; else frontier="$ch"; break; fi done if [ -z "$frontier" ]; then - echo "$cand - - COMPLETE 0 0 0 0 0 0 0 -"; return 0 + echo "$cand - - COMPLETE 0 0 0 0 0 0 0 - -"; return 0 fi local transition source cut_z now_epoch @@ -454,7 +585,7 @@ _frontier_state() { cut_z="$(candidate_cut_date "$agent" "$cand")" if [ -z "$cut_z" ]; then # Cannot determine the per-candidate window start — fail closed to prevent unbounded history queries. - echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 0 -"; return 0 + echo "$cand $frontier $transition BLOCKED 0 0 0 0 0 0 0 - -"; return 0 fi now_epoch="$(date -u +%s)" @@ -518,6 +649,23 @@ _frontier_state() { local state; state="$(decide_graduated "$dwell_h" "$dwell_floor" "$sample" "$target" "$waived" "$cum_fail" "$cum_startup")" + # Layer 2 (#668 increment 4): opt-in DECISION telemetry. When the reliability verdict is + # PROMOTE and the agent opts into gate.correctness, tally the candidate's decision mix vs a + # prior-version baseline; a gross distribution SHIFT HOLDS the promotion (BLOCKED, triage + # SUSPECT — the correctness variant, flagged by mix_shift). INSUFFICIENT/OK are no-ops, so an + # agent that produces no comparable signal degrades to reliability-only. The overlay fires + # ONLY from an otherwise-PROMOTE verdict — it never masks or bypasses a reliability BLOCK, and + # it never rolls back or auto-acts (worst case: a few hours of human latency). + local mix_shift="-" + if [ "$state" = "PROMOTE" ] && [ -n "$(_gate_field "$agent" correctness)" ]; then + local src_csv verdict + src_csv="$(IFS=,; echo "${src_repos[*]}")" + verdict="$(_correctness_verdict "$agent" "$cand" "$cut_z" "$src_csv")" + if [ "$verdict" = "SHIFT" ]; then + state="BLOCKED"; mix_shift="SHIFT" + fi + fi + # Layer 3 (#668 increment 3): opt-in human confirmation at a ring boundary. When the # reliability verdict is PROMOTE but the frontier transition sets require_confirmation, hold # in a distinct AWAITING_CONFIRMATION state — SOAKING-like: the scheduled promote-all never @@ -528,12 +676,16 @@ _frontier_state() { state="AWAITING_CONFIRMATION" fi + # Triage: a correctness SHIFT is SUSPECT (correctness variant, distinguished by mix_shift); + # any other BLOCK is classified from the failure evidence. local triage="-" - if [ "$state" = "BLOCKED" ]; then + if [ "$state" = "BLOCKED" ] && [ "$mix_shift" = "SHIFT" ]; then + triage="SUSPECT" + elif [ "$state" = "BLOCKED" ]; then triage="$(classify_failure "$differs" "${CANARY_FAILURE_CATEGORY:-unknown}" "$cum_suspect")" fi - echo "$cand $frontier $transition $state $dwell_h $dwell_floor $sample $target $cum_fail $cum_startup $cum_benign $triage" + echo "$cand $frontier $transition $state $dwell_h $dwell_floor $sample $target $cum_fail $cum_startup $cum_benign $triage $mix_shift" } cmd_evaluate() { @@ -549,7 +701,7 @@ cmd_evaluate() { local mark=" "; [ -n "$cand" ] && [ "$c" = "$cand" ] && mark="* " printf ' %s%-7s -> %s\n' "$mark" "$ch" "${c:0:12}" done - read -r _cand frontier transition state dwell floor sample target cum_fail cum_startup cum_benign triage < <(_frontier_state "$agent") + read -r _cand frontier transition state dwell floor sample target cum_fail cum_startup cum_benign triage mix_shift < <(_frontier_state "$agent") echo "----" if [ "$frontier" = "-" ]; then echo "frontier: none — fully rolled out (all rings on candidate)." @@ -559,6 +711,8 @@ cmd_evaluate() { if [ "$state" = "BLOCKED" ]; then if [ "$triage" = "REGRESSION" ]; then echo "::error::triage=REGRESSION — candidate changed the reusable and a run failed since cut. HALT + hold; recommend rollback." + elif [ "$triage" = "SUSPECT" ] && [ "$mix_shift" = "SHIFT" ]; then + echo "::warning::triage=SUSPECT (decision-mix shift, #668 Layer 2) — the candidate's decision distribution moved ≥ threshold vs the prior-version baseline. HOLDS the promotion; review the decision-mix table in the blocker issue. Composition drift (e.g. a draft-PR burst) is the usual false positive → promote --override to dismiss; a genuine always/never shift → roll back." elif [ "$triage" = "SUSPECT" ]; then echo "::warning::triage=SUSPECT — failure matches a suspect class (possibly candidate-caused). BLOCKS + needs a human; see the blocker issue's discriminating question, then promote --override if unrelated or roll back if a real regression." else @@ -601,7 +755,7 @@ cmd_promote() { *) echo "::error::unknown promote flag: $1" >&2; return 2 ;; esac; shift done - read -r cand frontier transition state _dwell _floor _sample _target cum_fail _cum_startup _cum_benign triage < <(_frontier_state "$agent") + read -r cand frontier transition state _dwell _floor _sample _target cum_fail _cum_startup _cum_benign triage _mix_shift < <(_frontier_state "$agent") if [ "$frontier" = "-" ]; then echo "nothing to promote — $agent is fully rolled out."; return 0 fi @@ -796,9 +950,35 @@ _suspect_guidance() { | "- **\(.id):** \(.guidance)"' } -# _blocker_body +# _blocker_body [] [] _blocker_body() { - local agent="$1" transition="$2" cand="$3" cum_fail="$4" cum_startup="$5" triage="$6" host="$7" evidence="$8" note + local agent="$1" transition="$2" cand="$3" cum_fail="$4" cum_startup="$5" triage="$6" host="$7" evidence="$8" mix_shift="${9:--}" mix_table="${10:-}" note + # Correctness variant (#668 L2): a decision-mix SHIFT holds as SUSPECT but is NOT a reliability + # failure — the candidate exited green yet its decision distribution moved. Distinct note + a + # candidate-vs-baseline mix table replace the failing-runs evidence (there are none). + if [ "$triage" = "SUSPECT" ] && [ "$mix_shift" = "SHIFT" ]; then + note="> ⚠️ **SUSPECT (decision-mix shift, #668 Layer 2)** — reliability PASSED (the candidate exits green), but its decision-class distribution shifted materially vs the prior version on comparable traffic. This HOLDS and needs a human (labelled \`needs-human\`): decide whether the shift is an intended behaviour change (\`promote --override\`) or a correctness regression (roll back). The gate never rolls back on its own." + cat < +**Automated canary-rollout blocker.** The release gate is holding \`$agent\` and will not promote it until this clears. Filed + maintained by the Canary Rollout workflow (gate standard: .github#548); this issue is **regenerated each run and auto-closes** when the gate passes — do not edit the table below by hand. + +| field | value | +|---|---| +| agent | \`$agent\` | +| transition | \`$transition\` | +| candidate | \`${cand:0:12}\` | +| host repo | \`$host\` | +| triage | **SUSPECT** (decision-mix shift) | + +$note + +### Decision-mix (candidate vs prior-version baseline) +${mix_table:-_(mix table unavailable — samples could not be resolved)_} +--- +_Whole-fleet status is in the Canary Rollout workflow run's job summary (Actions → Canary Rollout → latest run → Summary)._ +EOF + return 0 + fi if [ "$triage" = "REGRESSION" ]; then note="> ⛔ **REGRESSION** — the candidate changed the reusable and a run failed since its cut. HALT + hold; investigate and roll back rather than \`--override\`. (labelled \`needs-human\`)" elif [ "$triage" = "SUSPECT" ]; then @@ -926,8 +1106,8 @@ cmd_sync_issues() { local rows="" agent while IFS= read -r agent; do [ -z "$agent" ] && continue - local cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage host - read -r cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage < <(_frontier_state "$agent") + local cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage mix_shift host + read -r cand frontier transition state _d _f _s _t cum_fail cum_startup _cb triage mix_shift < <(_frontier_state "$agent") host="$(_agent_field "$agent" host)" local blk="—" num_state num istate # Best-effort (#1081): these substitutions call gh/jq. Under `set -euo pipefail` @@ -937,10 +1117,15 @@ cmd_sync_issues() { num_state="$(_issue_find canary-blocker "" || true)" num="${num_state%%$'\t'*}"; istate="${num_state##*$'\t'}" if [ "$state" = "BLOCKED" ]; then - local evidence body title + local evidence body title mix_table="" evidence="$(_blocker_evidence "$agent" "$cand" || true)" - body="$(_blocker_body "$agent" "$transition" "$cand" "$cum_fail" "$cum_startup" "$triage" "$host" "$evidence")" - title="Canary blocker: $agent $transition (cum_fail=$cum_fail, $triage)" + [ "$mix_shift" = "SHIFT" ] && mix_table="$(_decision_mix_table "$agent" "$cand" || true)" + body="$(_blocker_body "$agent" "$transition" "$cand" "$cum_fail" "$cum_startup" "$triage" "$host" "$evidence" "$mix_shift" "$mix_table")" + if [ "$mix_shift" = "SHIFT" ]; then + title="Canary blocker: $agent $transition (decision-mix shift, SUSPECT)" + else + title="Canary blocker: $agent $transition (cum_fail=$cum_fail, $triage)" + fi if [ -z "$num" ]; then if [ "$dry" = true ]; then echo " [DRY] would OPEN blocker issue for $agent ($triage)"; blk="(new)"; else num="$(_gh_issue_create "$title" "$body" "canary-blocker" || true)" diff --git a/scripts/lib/canary-rollout.sh b/scripts/lib/canary-rollout.sh index 4729f2ddd..e1e4e111c 100644 --- a/scripts/lib/canary-rollout.sh +++ b/scripts/lib/canary-rollout.sh @@ -257,3 +257,77 @@ gate_summary_line() { printf '%-14s %-9s dwell=%sh/%sh sample=%s/%s cum_fail=%s startup=%s benign=%s\n' \ "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" "${9:-0}" } + +# ── decision telemetry: gross decision-mix shift (#668 Layer 2, increment 4) ──── +# A canary doesn't need the *right* answer — it needs to detect the candidate answering +# DIFFERENTLY than the incumbent on comparable traffic. Both #668 pathologies (dispatching +# when it shouldn't; never dispatching) are DISTRIBUTION pathologies: a decision class's +# share swings toward ~100% or ~0% vs a stable-tier baseline. These pure cores tally the +# decision mix (from emitted `decision: ` step names) and decide a gross shift. +# jq is used as a pure, deterministic transform only — no gh/git/network I/O. + +# decision_class — the decision class a run TOOK: the first step +# whose name starts with AND did not skip (conclusion != "skipped"), with the prefix +# stripped. is the `gh run view --json jobs` payload; non-`` and +# skipped steps are ignored (the reusable emits one `decision: ` no-op step per outcome +# branch, only the taken branch runs). Empty when no decision step ran (absent/unparseable). +decision_class() { + local prefix="$1" json="$2" + [ -z "$json" ] && json='{}' + jq -r --arg p "$prefix" ' + [ (.jobs[]?.steps[]?) + | select((.conclusion // "") != "skipped") + | (.name // "") + | select(startswith($p)) + | .[($p|length):] ] + | .[0] // "" + ' <<< "$json" 2>/dev/null || echo "" +} + +# decide_decision_shift — gate the +# candidate's decision mix against a prior-version baseline. Counts are objects of +# decision-class → run count, e.g. {"dispatched":6,"skip-draft":4}. Knobs: +# {min_candidate_sample, min_baseline_sample, max_shift_permille}. Echoes exactly one of: +# INSUFFICIENT — either side is below its min, or a side is empty/unparseable (no decision +# steps produced) → NO EFFECT: the caller degrades to reliability-only, so a +# low-traffic agent never stalls on a signal it cannot produce. +# SHIFT — some decision class's share (per-mille of that side's total) moved by at +# least max_shift_permille between candidate and baseline → HOLD (SUSPECT). +# OK — every class's share is within threshold → no effect. +# Shares are integer per-mille, rounded half-up; the max absolute per-class delta is compared +# to max_shift_permille (inclusive). Gross always/never/gross shifts only — by design (n=10–25). +decide_decision_shift() { + local cand="$1" base="$2" knobs="$3" + [ -z "$cand" ] && cand='{}' + [ -z "$base" ] && base='{}' + [ -z "$knobs" ] && knobs='{}' + local min_c min_b max_shift + min_c="$(jq -r '.min_candidate_sample // 0' <<< "$knobs" 2>/dev/null || echo 0)" + min_b="$(jq -r '.min_baseline_sample // 0' <<< "$knobs" 2>/dev/null || echo 0)" + max_shift="$(jq -r '.max_shift_permille // 0' <<< "$knobs" 2>/dev/null || echo 0)" + local cand_total base_total + cand_total="$(jq -r 'if type=="object" then ([.[]?]|add // 0) else "x" end' <<< "$cand" 2>/dev/null || echo x)" + base_total="$(jq -r 'if type=="object" then ([.[]?]|add // 0) else "x" end' <<< "$base" 2>/dev/null || echo x)" + # Unparseable / non-object / empty → INSUFFICIENT (no decision steps to compare). + case "$cand_total" in ''|*[!0-9]*) echo "INSUFFICIENT"; return 0 ;; esac + case "$base_total" in ''|*[!0-9]*) echo "INSUFFICIENT"; return 0 ;; esac + if [ "$cand_total" -lt "$min_c" ] || [ "$base_total" -lt "$min_b" ]; then + echo "INSUFFICIENT"; return 0 + fi + if [ "$cand_total" -eq 0 ] || [ "$base_total" -eq 0 ]; then + echo "INSUFFICIENT"; return 0 + fi + # Max absolute per-class share delta, in integer per-mille (round half-up), over the union + # of decision classes seen on either side (a class absent on one side counts as 0 there). + local max_delta + max_delta="$(jq -nr --argjson c "$cand" --argjson b "$base" \ + --argjson ct "$cand_total" --argjson bt "$base_total" ' + (($c|keys) + ($b|keys) | unique) as $classes + | [ $classes[] + | ((( ($c[.]//0)*1000 + ($ct/2) ) / $ct) | floor) as $cs + | ((( ($b[.]//0)*1000 + ($bt/2) ) / $bt) | floor) as $bs + | (($cs - $bs) | if . < 0 then -. else . end) ] + | max // 0' 2>/dev/null || echo 0)" + case "$max_delta" in ''|*[!0-9]*) max_delta=0 ;; esac + if [ "$max_delta" -ge "$max_shift" ]; then echo "SHIFT"; else echo "OK"; fi +} diff --git a/standards/canary-rings.json b/standards/canary-rings.json index 5d6638ad7..95d6b082e 100644 --- a/standards/canary-rings.json +++ b/standards/canary-rings.json @@ -902,6 +902,13 @@ "allow_pre_existing": false }, "_benign_note": "benign_failure_classes (#1025 P2): per-reusable allowlist matched against a failed run's workflow name (`workflow` ERE) + failed-step signature (`step` ERE, matched via `gh run view`). Matches are excluded from cum_fail \u2014 but ONLY when the candidate's reusable is byte-identical to the prior channel (differs=0), so the allowlist can never mask a candidate-introduced regression. Exception (#668): a class marked `version_independent: true` fails before/independent of the candidate's own code (e.g. a Dependabot-context secrets failure at startup) and can NEVER be candidate-caused, so it is excluded even when differs=1 \u2014 reserve the flag for classes where that is provable from the failure mechanism, not merely likely. `control.allow_pre_existing`, or `promote --allow-pre-existing`, advances a BLOCKED+PRE_EXISTING frontier (never REGRESSION).", + "correctness": { + "decision_step_prefix": "decision: ", + "min_candidate_sample": 10, + "min_baseline_sample": 20, + "max_shift_permille": 400 + }, + "_correctness_note": "correctness (#668 increment 4, Layer 2): opt-in DECISION telemetry on organic runs \u2014 the actual correctness signal. pr-auto-review emits one no-op `decision: ` step per outcome branch (dispatched / skip-draft / skip-checks-pending / skip-changes-requested / skip-unresolved-threads); only the taken branch runs, the rest skip. The engine tallies the candidate's decision mix on the source tier (per-candidate window) against a prior-version baseline (source tier, pre-cut trailing window) \u2014 both read from `gh run view --json jobs` step names, prefix-matched by `decision_step_prefix`. `decide_decision_shift` returns OK|SHIFT|INSUFFICIENT: INSUFFICIENT (either sample below its min, or absent/unparseable decision steps) is a NO-OP that degrades to reliability-only, so low-traffic agents never stall on a signal they can't produce; SHIFT (any decision class's share moves \u2265 max_shift_permille per-mille) HOLDS the promotion with triage SUSPECT (correctness variant) and sync-issues renders a candidate-vs-baseline decision-mix table in the blocker issue; OK is a no-op. Never rolls back, never auto-acts \u2014 SHIFT only holds (worst case: a few hours of human latency). Gross always/never/gross shifts only, by design (n=10\u201325). Default-absent (this key omitted) = today's behaviour, byte-identical for agents that do not opt in.", "transitions": { "next->ring0": { "dwell_hours": 4, diff --git a/tests/canary_rollout.bats b/tests/canary_rollout.bats index e7487f00f..b80045854 100644 --- a/tests/canary_rollout.bats +++ b/tests/canary_rollout.bats @@ -1752,3 +1752,240 @@ GHEOF [[ "$output" == *"closed cleared confirm issue #502 for dev-lead"* ]] grep -q "CLOSE|.*502" "$ISSUE_LOG" } + +# ── #668 increment 4 (Layer 2): decision telemetry — pure core + engine overlay ── +# decision_class(): the taken `decision: ` no-op step (skipped branches ignored, prefix +# stripped) off a `gh run view --json jobs` payload. decide_decision_shift(): the pure gate over +# a candidate vs prior-version decision-mix. Both are pure (no gh/git) → sourced $LIB, no stubs. +@test "decision_class: returns the taken decision (skipped branches ignored, prefix stripped)" { + json='{"jobs":[{"steps":[ + {"name":"Resolve PR URL","conclusion":"success"}, + {"name":"decision: dispatched","conclusion":"skipped"}, + {"name":"decision: skip-draft","conclusion":"skipped"}, + {"name":"decision: skip-checks-pending","conclusion":"success"} + ]}]}' + [ "$(decision_class 'decision: ' "$json")" = "skip-checks-pending" ] +} +@test "decision_class: a run with no decision step → empty (degrades to INSUFFICIENT upstream)" { + [ "$(decision_class 'decision: ' '{"jobs":[{"steps":[{"name":"Build","conclusion":"success"}]}]}')" = "" ] +} +@test "decision_class: empty / absent / non-object json → empty (never errs)" { + [ "$(decision_class 'decision: ' '')" = "" ] + [ "$(decision_class 'decision: ' '{}')" = "" ] + [ "$(decision_class 'decision: ' 'not json')" = "" ] +} + +DECISION_KNOBS='{"min_candidate_sample":10,"min_baseline_sample":20,"max_shift_permille":400}' +@test "decide_decision_shift: a gross share move ≥ threshold → SHIFT" { + # candidate is all skip-checks-pending (0‰ dispatched) vs an all-dispatched baseline → 1000‰ move + [ "$(decide_decision_shift '{"skip-checks-pending":12}' '{"dispatched":22}' "$DECISION_KNOBS")" = "SHIFT" ] +} +@test "decide_decision_shift: shares within threshold → OK (no effect)" { + # ~917‰/83‰ vs ~909‰/91‰ dispatched/skip-draft — max per-class delta 8‰ ≪ 400‰ + [ "$(decide_decision_shift '{"dispatched":11,"skip-draft":1}' '{"dispatched":20,"skip-draft":2}' "$DECISION_KNOBS")" = "OK" ] +} +@test "decide_decision_shift: a move exactly at max_shift_permille → SHIFT (inclusive)" { + # 500‰ dispatched (cand) vs 900‰ (base) → delta 400‰ == threshold + [ "$(decide_decision_shift '{"dispatched":10,"skip-draft":10}' '{"dispatched":18,"skip-draft":2}' "$DECISION_KNOBS")" = "SHIFT" ] +} +@test "decide_decision_shift: candidate below min_candidate_sample → INSUFFICIENT" { + [ "$(decide_decision_shift '{"dispatched":5}' '{"dispatched":22}' "$DECISION_KNOBS")" = "INSUFFICIENT" ] +} +@test "decide_decision_shift: baseline below min_baseline_sample → INSUFFICIENT" { + [ "$(decide_decision_shift '{"dispatched":12}' '{"dispatched":5}' "$DECISION_KNOBS")" = "INSUFFICIENT" ] +} +@test "decide_decision_shift: empty / unparseable side → INSUFFICIENT (no decision steps to compare)" { + [ "$(decide_decision_shift '{}' '{"dispatched":22}' "$DECISION_KNOBS")" = "INSUFFICIENT" ] + [ "$(decide_decision_shift '{"dispatched":12}' '' "$DECISION_KNOBS")" = "INSUFFICIENT" ] + [ "$(decide_decision_shift 'garbage' '{"dispatched":22}' "$DECISION_KNOBS")" = "INSUFFICIENT" ] +} + +@test "canary-rings.json: pr-auto-review opts into gate.correctness (#668 L2); no other agent does" { + run jq -e '.agents["pr-auto-review"].gate.correctness | .decision_step_prefix=="decision: " and .min_candidate_sample==10 and .min_baseline_sample==20 and .max_shift_permille==400' "$RINGS" + [ "$status" -eq 0 ] + # default-off everywhere else: pr-auto-review is the ONLY agent carrying a correctness block + run bash -c "jq -r '[.agents|to_entries[]|select(.value.gate.correctness)|.key]|sort|join(\",\")' '$RINGS'" + [ "$output" = "pr-auto-review" ] +} + +# The reusable emits one `decision: ` no-op step per outcome branch — the engine reads +# their names off `gh run view --json jobs` (decision_class). Structural guard on the workflow. +@test "pr-auto-review-reusable: emits a decision no-op step per outcome branch + writes the output" { + local wf="$SCRIPT_DIR/.github/workflows/pr-auto-review-reusable.yml" c + for c in dispatched skip-draft skip-checks-pending skip-changes-requested skip-unresolved-threads; do + run grep -F "name: 'decision: $c'" "$wf" + [ "$status" -eq 0 ] + run grep -F "decision=$c" "$wf" + [ "$status" -eq 0 ] + done + # each decision step is a side-effect-free no-op + run grep -c "run: 'true'" "$wf" + [ "$output" -ge 5 ] +} + +# ── #668 L2: engine — sample the decision mix + overlay the gate (with gh stubs) ─ +# dev-lead layout (next=cand cccc, rings=prior bbbb, cut 3d ago, reusable identical → differs=0) +# PLUS a decision-mix sample: source-tier (next=.github-private) CANDIDATE runs (createdAt 1d ago, +# ids ≥2000, class=) and prior-version BASELINE runs (createdAt 7d ago, ids <2000, +# class=). `run view` returns the taken decision no-op step for a run id. gate.correctness +# is injected onto the dev-lead clone in $CORR_RINGS so the opt-in overlay fires. +_correctness_stub() { + local cand_class="$1" cand_n="$2" base_class="$3" base_n="$4" + STUB_BIN="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"; export PATH="$STUB_BIN:$PATH" + local cut_iso cand_iso base_iso + cut_iso="$(date -u -d '-3 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-3d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cand_iso="$(date -u -d '-1 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + base_iso="$(date -u -d '-7 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-7d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/git" <<'GITEOF' +#!/usr/bin/env bash +: # dev-lead is cross-repo; all tag/blob resolution goes via gh api +GITEOF + chmod +x "$STUB_BIN/git" + cat > "$STUB_BIN/gh" </dev/null && cls="$cand_class" + jq -nc --arg cls "\$cls" '{jobs:[{steps:[ + {name:"Resolve PR URL",conclusion:"success"}, + {name:"decision: skip-unresolved-threads",conclusion:"skipped"}, + {name:("decision: "+\$cls),conclusion:"success"} + ]}]}' ;; + *"run list"*) + since=""; prev="" + for a in "\$@"; do [ "\$prev" = "--created" ] && since="\$a"; prev="\$a"; done + since="\${since#>=}" + jq -nc --arg s "\$since" --arg cc "$cand_iso" --arg bb "$base_iso" \ + --argjson cn "$cand_n" --argjson bn "$base_n" ' + ( [range(2001;2001+\$cn)|{databaseId:.,conclusion:"success",createdAt:\$cc,workflowName:"Dev-Lead Agent"}] + + [range(1001;1001+\$bn)|{databaseId:.,conclusion:"success",createdAt:\$bb,workflowName:"Dev-Lead Agent"}] ) + | map(select(\$s=="" or .createdAt >= \$s))' ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + CORR_RINGS="$BATS_TEST_TMPDIR/corr-rings.json" + jq '.agents["dev-lead"].gate.correctness = {decision_step_prefix:"decision: ",min_candidate_sample:10,min_baseline_sample:20,max_shift_permille:400}' "$RINGS" > "$CORR_RINGS" +} + +_CORR_CAND="cccccccccccccccccccccccccccccccccccccccc" +_corr_cut() { date -u -d '-3 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-3d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null; } + +@test "_correctness_verdict: a gross candidate decision-mix shift → SHIFT" { + _correctness_stub skip-checks-pending 20 dispatched 22 + run env CANARY_RINGS="$CORR_RINGS" bash -c "source '$ORCH' && _correctness_verdict dev-lead $_CORR_CAND '$(_corr_cut)' petry-projects/.github-private" + [ "$status" -eq 0 ]; [ "$output" = "SHIFT" ] +} +@test "_correctness_verdict: candidate mix matches the baseline → OK (no effect)" { + _correctness_stub dispatched 20 dispatched 22 + run env CANARY_RINGS="$CORR_RINGS" bash -c "source '$ORCH' && _correctness_verdict dev-lead $_CORR_CAND '$(_corr_cut)' petry-projects/.github-private" + [ "$status" -eq 0 ]; [ "$output" = "OK" ] +} +@test "_correctness_verdict: candidate sample below the minimum → INSUFFICIENT (no-op)" { + _correctness_stub dispatched 5 dispatched 22 + run env CANARY_RINGS="$CORR_RINGS" bash -c "source '$ORCH' && _correctness_verdict dev-lead $_CORR_CAND '$(_corr_cut)' petry-projects/.github-private" + [ "$status" -eq 0 ]; [ "$output" = "INSUFFICIENT" ] +} + +@test "orchestrator: evaluate — a decision-mix SHIFT holds an otherwise-PROMOTE candidate as BLOCKED/SUSPECT (#668 L2)" { + _correctness_stub skip-checks-pending 20 dispatched 22 + run env CANARY_RINGS="$CORR_RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"next->ring0"* ]] + [[ "$output" == *"BLOCKED"* ]] + [[ "$output" == *"SUSPECT"* ]] + [[ "$output" == *"decision-mix shift"* ]] +} +@test "orchestrator: evaluate — an in-threshold decision mix leaves the PROMOTE verdict untouched (#668 L2)" { + _correctness_stub dispatched 20 dispatched 22 + run env CANARY_RINGS="$CORR_RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"PROMOTE"* ]] + ! [[ "$output" == *"decision-mix shift"* ]] +} +@test "orchestrator: evaluate — an agent WITHOUT gate.correctness never runs the overlay (byte-identical, #668 L2)" { + # Same PROMOTE layout, default registry (dev-lead has no correctness block) → no sampling, no SUSPECT. + _graduated_stub 3 2 success 0 + run env CANARY_RINGS="$RINGS" bash "$ORCH" evaluate dev-lead + [ "$status" -eq 0 ] + [[ "$output" == *"PROMOTE"* ]] + ! [[ "$output" == *"decision-mix"* ]] + ! [[ "$output" == *"SUSPECT"* ]] +} + +# sync-issues renders the candidate-vs-baseline decision-mix table into the blocker body for a +# correctness SHIFT (dev-lead-only registry keeps the fleet loop to one agent; gh logs issue ops). +_correctness_sync_stub() { + local cand_class="$1" cand_n="$2" base_class="$3" base_n="$4" blocker_list="${5:-[]}" + _correctness_stub "$cand_class" "$cand_n" "$base_class" "$base_n" + export ISSUE_LOG="$STUB_BIN/issue.log"; : > "$ISSUE_LOG" + # Extend the gh stub with issue ops (append the new cases BEFORE the catch-all). + local cut_iso cand_iso base_iso + cut_iso="$(_corr_cut)" + cand_iso="$(date -u -d '-1 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-1d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + base_iso="$(date -u -d '-7 days' +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || date -u -v-7d +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" + cat > "$STUB_BIN/gh" </dev/null && cls="$cand_class" + jq -nc --arg cls "\$cls" '{jobs:[{steps:[ + {name:"Resolve PR URL",conclusion:"success"}, + {name:"decision: skip-unresolved-threads",conclusion:"skipped"}, + {name:("decision: "+\$cls),conclusion:"success"} + ]}]}' ;; + *"run list"*) + since=""; prev="" + for a in "\$@"; do [ "\$prev" = "--created" ] && since="\$a"; prev="\$a"; done + since="\${since#>=}" + jq -nc --arg s "\$since" --arg cc "$cand_iso" --arg bb "$base_iso" \ + --argjson cn "$cand_n" --argjson bn "$base_n" ' + ( [range(2001;2001+\$cn)|{databaseId:.,conclusion:"success",createdAt:\$cc,workflowName:"Dev-Lead Agent"}] + + [range(1001;1001+\$bn)|{databaseId:.,conclusion:"success",createdAt:\$bb,workflowName:"Dev-Lead Agent"}] ) + | map(select(\$s=="" or .createdAt >= \$s))' ;; + "issue list"*) echo '$blocker_list' ;; + "issue create"*) echo "CREATE|\$*" >> "$ISSUE_LOG"; echo "https://github.com/petry-projects/.github-private/issues/777" ;; + "issue edit"*) echo "EDIT|\$*" >> "$ISSUE_LOG" ;; + "issue close"*) echo "CLOSE|\$*" >> "$ISSUE_LOG" ;; + "issue reopen"*) echo "REOPEN|\$*" >> "$ISSUE_LOG" ;; + "label create"*) : ;; + *) echo "{}" ;; +esac +GHEOF + chmod +x "$STUB_BIN/gh" + # dev-lead-only registry (with the correctness block) so the fleet loop stays a single agent. + CORR_RINGS="$BATS_TEST_TMPDIR/corr-sync-rings.json" + jq '{org_infra_repos, agents: {"dev-lead": (.agents["dev-lead"] | .gate.correctness = {decision_step_prefix:"decision: ",min_candidate_sample:10,min_baseline_sample:20,max_shift_permille:400})}}' "$RINGS" > "$CORR_RINGS" +} + +@test "orchestrator: sync-issues renders the decision-mix table in the blocker for a correctness SHIFT (#668 L2)" { + _correctness_sync_stub skip-checks-pending 20 dispatched 22 '[]' + run env CANARY_RINGS="$CORR_RINGS" ISSUE_REPO="petry-projects/.github-private" bash "$ORCH" sync-issues + [ "$status" -eq 0 ] + [[ "$output" == *"opened blocker issue #777 for dev-lead"* ]] + # The blocker body (passed to `gh issue create --body`) carries the correctness note + mix table. + grep -q "decision-mix shift" "$ISSUE_LOG" + grep -q "decision class" "$ISSUE_LOG" + grep -q "candidate" "$ISSUE_LOG" + grep -q "skip-checks-pending" "$ISSUE_LOG" + # SUSPECT → routed to a human (needs-human) as well as the dev-lead agent. + grep -q -- "--add-label needs-human" "$ISSUE_LOG" +} From d45c763e99d1139a4767e7d3171777008f7fc297 Mon Sep 17 00:00:00 2001 From: Claude Code Bot Date: Sun, 12 Jul 2026 13:57:08 -0500 Subject: [PATCH 2/2] =?UTF-8?q?fix(canary=20#683):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20single-pass=20jq=20counts,=20empty-argjson=20guard,?= =?UTF-8?q?=20drop=20out-of-scope=20lockfiles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _sample_decision_counts: build the counts object in ONE jq pass instead of spawning jq per key (gemini @488) — jq handles escaping so class names stay valid. - _decision_mix_table: guard cand_counts/base_counts before --argjson (gemini @545). NOTE the suggested `${var:-{}}` inline default is unusable — the inner `}` closes the parameter expansion and appends a stray brace → invalid JSON; use explicit guards. - Revert accidental package-lock.json / node_modules "name" churn (out of scope for the decision-telemetry change; also the source of the DIRTY merge conflict). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01J4z5uYVjwdyQjh2wFZxkut --- node_modules/.package-lock.json | 2 +- package-lock.json | 2 +- scripts/canary-rollout.sh | 15 +++++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json index 5e396f0fc..e815605ad 100644 --- a/node_modules/.package-lock.json +++ b/node_modules/.package-lock.json @@ -1,5 +1,5 @@ { - "name": ".github", + "name": "pr-681", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/package-lock.json b/package-lock.json index 6f0776ff0..060eb1523 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": ".github", + "name": "pr-681", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/scripts/canary-rollout.sh b/scripts/canary-rollout.sh index 48095e169..d65485f67 100755 --- a/scripts/canary-rollout.sh +++ b/scripts/canary-rollout.sh @@ -482,10 +482,12 @@ _sample_decision_counts() { sampled=$(( sampled + 1 )) done < <(jq -r "[${filter}]|sort_by(.createdAt)|reverse|.[]|(.databaseId|tostring)" 2>/dev/null <<< "$json") done - local out='{}' k - for k in "${!counts[@]}"; do - out="$(jq -c --arg k "$k" --argjson v "${counts[$k]}" '. + {($k):$v}' <<< "$out")" - done + # Build the counts object in a SINGLE jq pass (one process, not one per key) — + # jq does the key/value escaping so arbitrary class names stay valid JSON. + local out k + out="$(for k in "${!counts[@]}"; do printf '%s\t%s\n' "$k" "${counts[$k]}"; done \ + | jq -Rn '[inputs | split("\t") | {(.[0]): (.[1] | tonumber)}] | add // {}' 2>/dev/null)" + [ -n "$out" ] || out='{}' echo "$out" } @@ -542,6 +544,11 @@ _decision_mix_table() { base_since="$(_iso_now_minus_days "$win")" cand_counts="$(_sample_decision_counts "$agent" "$prefix" 15 "$cut_z" "-" "${src_repos[@]}")" base_counts="$(_sample_decision_counts "$agent" "$prefix" 25 "$base_since" "$cut_z" "${src_repos[@]}")" + # Guard against an empty sample string — --argjson rejects empty input. + # (A `${var:-{}}` inline default is NOT usable: the inner `}` closes the + # parameter expansion, appending a stray brace and producing invalid JSON.) + [ -n "$cand_counts" ] || cand_counts='{}' + [ -n "$base_counts" ] || base_counts='{}' jq -nr --argjson c "$cand_counts" --argjson b "$base_counts" --arg win "$win" ' ([$c[]?]|add // 0) as $ct | ([$b[]?]|add // 0) as $bt | (($c|keys) + ($b|keys) | unique) as $classes