CI hardening: openipc-bisect stability + enrich_manifest retry

.github/scripts/enrich_manifest.py

@@ -22,20 +22,46 @@
 import re
 import subprocess
 import sys
+import time
 from pathlib import Path
 
 REPO = os.environ.get("GITHUB_REPOSITORY", "OpenIPC/firmware")
 RETENTION = 90
 TAG_RE = re.compile(r"^nightly-(\d{8})-([0-9a-f]{7})$")
 ASSET_RE = re.compile(r"^openipc\.([^.]+)-(nor|nand)-(lite|ultimate|neo)\.tgz$")
 
+# Retry budget for transient GitHub API failures (HTTP 401 Bad credentials,
+# 5xx, rate-limit) observed on workflow_run-triggered runs 2026-05-23.
+GH_RETRY_DELAYS = (0, 5, 15, 40)  # 4 attempts; last delay before final try
+
 
 def gh(*args: str) -> str:
     # Always pass --repo so we don't depend on a .git in cwd
     # (the workflow runs the script from a path without .git).
-    return subprocess.check_output(
-        ["gh", *args, "--repo", REPO], text=True
+    # Retry on transient failures (the GitHub API/token plane has flaky days);
+    # surface to stderr so failures are visible in the action log.
+    cmd = ["gh", *args, "--repo", REPO]
+    last_exc = None
+    for delay in GH_RETRY_DELAYS:
+        if delay > 0:
+            time.sleep(delay)
+        try:
+            return subprocess.check_output(cmd, text=True, stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError as e:
+            last_exc = e
+            err = e.stderr or ""
+            # Permanent failures — don't waste the retry budget.
+            if "release not found" in err.lower() or "not found (HTTP 404)" in err:
+                break
+            sys.stderr.write(
+                f"gh {' '.join(args[:3])}: attempt failed "
+                f"(rc={e.returncode}): {err.strip()[:240]}\n"
+            )
+    sys.stderr.write(
+        f"gh {' '.join(args[:3])}: giving up; "
+        f"final stderr:\n{last_exc.stderr}\n"
     )
+    raise last_exc
 
 
 def list_dated_releases() -> list[dict]:

contrib/openipc-bisect

@@ -27,7 +27,7 @@ PROG=$(basename "$0")
 MANIFEST_URL=${OPENIPC_MANIFEST_URL:-https://openipc.github.io/firmware/manifest.json}
 STATE_DIR=${OPENIPC_BISECT_STATE:-${XDG_STATE_HOME:-$HOME/.local/state}/openipc/bisect}
 WAIT_BUDGET=${OPENIPC_BISECT_WAIT:-300}   # seconds to wait for a camera to come back
-SSH_OPTS=${OPENIPC_SSH_OPTS:--o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new}
+SSH_OPTS=${OPENIPC_SSH_OPTS:--o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 -o ServerAliveCountMax=3}
 
 die() { printf '%s: %s\n' "$PROG" "$*" >&2; exit 1; }
 info() { printf '%s\n' "$*" >&2; }
@@ -146,7 +146,7 @@ pick_next() {
 		.window as $w |
 		(.verdicts // {}) as $v |
 		($w | map(select($v[.] == null))) as $unverified |
-		if ($unverified | length) <= 1 then "" else
+		if ($unverified | length) == 0 then "" else
 			$unverified[($unverified | length / 2 | floor)]
 		end
 	'
@@ -261,6 +261,10 @@ iterate() {
 
 cmd_start() {
 	host=$1; shift
+	# Tolerate user@host (the form most OpenIPC docs use, e.g. root@cam).
+	# The script always SSHes as root because that's the only user on these
+	# cameras; the user@ prefix is a doc convention, not a real choice.
+	host=${host#*@}
 	good_ref=""; bad_ref=""; platform=""
 	while [ $# -gt 0 ]; do
 		case "$1" in
@@ -349,12 +353,15 @@ cmd_verdict() {
 cmd_status() {
 	host=$(resolve_host)
 	state=$(load_state "$host")
-	printf '%s\n' "$state" | jq '
+	wn=$(printf '%s' "$state" | jq -r '.window | length')
+	# jq has no log() — compute ceil(log2(N)) in awk.
+	rounds=$(awk -v n="$wn" 'BEGIN { if (n>1) { r=0; x=n; while (x>1) { r++; x=(x+1)/2 } print r } else print 0 }')
+	printf '%s\n' "$state" | jq --argjson rl "$rounds" '
 		{
 			host, platform, good, bad, current,
 			window_size: (.window | length),
-			verdicts,
-			est_rounds_left: ((.window | length) | (if . > 1 then (log/log(2)) | floor + 1 else 0 end))
+			est_rounds_left: $rl,
+			verdicts
 		}'
 }