fix(strix): retry artifact-less provider crashes without weakening exhaustion

.github/workflows/strix.yml

@@ -807,7 +807,7 @@ jobs:
           STRIX_LLM_MAX_RETRIES: 1
           STRIX_TRANSIENT_RETRY_PER_MODEL: 2
           STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60
-          STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openrouter' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }}
+          STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openrouter' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia_nim/nvidia/nemotron-3-super-120b-a12b github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }}
           STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }}
           STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }}
           STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"
@@ -835,50 +835,20 @@ jobs:
           export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds"
           export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700"
 
-          # Capture the gate exit code plus its console output. The gate returns
-          # exit 1 both for genuine blocking vulnerabilities AND for
-          # LLM-backend-unavailable outcomes (GitHub Models "Too many requests"
-          # rate limits, OpenAI quota starvation, 413 tokens_limit_reached
-          # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI
-          # infrastructure noise, not a security finding, so it must not fail
-          # the required check and block merges.
+          # Capture the gate exit code plus its console output so the always-run
+          # artifact steps can preserve exact provider and finding evidence.
+          # A provider outage without a vulnerability report is incomplete
+          # security evidence, so the gate's nonzero result must remain nonzero.
           strix_run_log="$RUNNER_TEMP/strix_gate_console.log"
           strix_rc=0
           set +e
           bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log"
           strix_rc="${PIPESTATUS[0]}"
           set -e
 
-          if [ "$strix_rc" -eq 0 ]; then
-            exit 0
-          fi
-
-          # Preserve configuration failures (exit 2) and any unexpected exit
-          # code as hard failures — only the scan-failure code (1) can be an
-          # infrastructure/backend-unavailability outcome.
-          if [ "$strix_rc" -ne 1 ]; then
-            exit "$strix_rc"
+          if [ "$strix_rc" -ne 0 ]; then
+            echo "::error title=Strix security gate failed::Strix returned exit ${strix_rc}. Provider outages without a vulnerability report are incomplete security evidence and are not converted to success. See the strix-reports artifact and run log."
           fi
-
-          # Recognized signals that the LLM backend was unavailable / starved.
-          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure'
-          # Any evidence that a vulnerability was actually reported. Its presence
-          # forces a hard failure so real findings are NEVER downgraded. Keep the
-          # severity branch anchored away from identifiers so environment lines
-          # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
-          reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
-
-          # Neutral skip only when ALL hold: a backend-unavailability signal is
-          # present and no vulnerability was reported anywhere. This preserves
-          # real security gating while keeping uncontrollable provider outages
-          # from blocking current-head merge progress.
-          if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \
-            && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then
-            echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log."
-            exit 0
-          fi
-
-          echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2
           exit "$strix_rc"
 
       - name: Collect Strix reports for artifact upload

scripts/ci/strix_quick_gate.sh

@@ -50,6 +50,7 @@ RUN_START_EPOCH=0
 TOTAL_TIMEOUT_EXCEEDED=0
 ATTEMPT_LOG_SEQUENCE=0
 PREEXISTING_REPORT_DIRS=()
+ATTEMPT_PREEXISTING_REPORT_RUN_DIRS=()
 REPO_NAME="${REPO_ROOT##*/}"
 # shellcheck source=scripts/ci/strix_model_utils.sh
 # shellcheck disable=SC1091  # source path is repo-local; local lint may omit -x
@@ -202,14 +203,55 @@ PY
 	done
 }
 
+# Snapshot the report run directories that already exist before a strix
+# attempt starts, so the per-attempt failure-signal check does not
+# re-attribute an earlier attempt's crash artifacts to the current attempt.
+# Earlier-attempt infrastructure errors stay recorded in the sticky
+# INFRA_ERROR_DETECTED flag, and vulnerability report artifacts from every
+# attempt remain visible to the severity checks regardless of this snapshot.
+capture_attempt_preexisting_report_run_dirs() {
+	ATTEMPT_PREEXISTING_REPORT_RUN_DIRS=()
+	local report_root run_dir
+	for report_root in "$@"; do
+		if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then
+			continue
+		fi
+		for run_dir in "$report_root"/*; do
+			if [ -d "$run_dir" ] && [ ! -L "$run_dir" ]; then
+				ATTEMPT_PREEXISTING_REPORT_RUN_DIRS+=("$run_dir")
+			fi
+		done
+	done
+}
+
+is_attempt_preexisting_report_run_dir() {
+	local candidate="$1"
+	local run_dir
+	for run_dir in "${ATTEMPT_PREEXISTING_REPORT_RUN_DIRS[@]}"; do
+		if [ "$candidate" = "$run_dir" ]; then
+			return 0
+		fi
+	done
+	return 1
+}
+
 has_strix_report_failure_signal() {
 	local report_root
 	local report_log
+	local attempt_run_dir
 	for report_root in "$@"; do
 		if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then
 			continue
 		fi
 		while IFS= read -r -d '' report_log; do
+			# ponytail: the snapshot assumes strix creates a fresh run
+			# directory per attempt; a log appended into an earlier
+			# attempt's run dir stays attributed to that earlier attempt.
+			attempt_run_dir="${report_log#"$report_root"/}"
+			attempt_run_dir="$report_root/${attempt_run_dir%%/*}"
+			if [ -d "$attempt_run_dir" ] && is_attempt_preexisting_report_run_dir "$attempt_run_dir"; then
+				continue
+			fi
 			if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then
 				return 0
 			fi
@@ -2349,6 +2391,7 @@ run_strix_once() {
 	if ! resolved_target_path="$(resolve_current_target_path "$TARGET_PATH")"; then
 		return 1
 	fi
+	capture_attempt_preexisting_report_run_dirs "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs"
 	local start_epoch
 	start_epoch="$(date +%s)"
 	local child_llm_api_key=""
@@ -3119,6 +3162,46 @@ has_blocking_vulnerability_reports() {
 	return 1
 }
 
+# Whether any attempt in this pipeline run produced an actual
+# vulnerabilities/*.md report artifact (report directories that pre-existed
+# the gate run are excluded).
+has_any_vulnerability_report_artifact() {
+	local run_dir vulnerabilities_dir vuln_file
+	for run_dir in "$STRIX_REPORTS_DIR"/*; do
+		if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then
+			continue
+		fi
+		if is_preexisting_report_dir "$run_dir"; then
+			continue
+		fi
+		vulnerabilities_dir="$run_dir/vulnerabilities"
+		if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then
+			continue
+		fi
+		for vuln_file in "$vulnerabilities_dir"/*.md; do
+			if [ -f "$vuln_file" ] && [ ! -L "$vuln_file" ]; then
+				return 0
+			fi
+		done
+	done
+	return 1
+}
+
+# Provider infrastructure crashes can echo severity-like markers into the
+# console log without writing any vulnerability report artifact. Log-only
+# markers are incomplete evidence, so such failures are retryable across
+# fallback models instead of non-recoverable. Any attempt that produced an
+# actual vulnerabilities/*.md artifact keeps failing closed.
+is_infra_error_without_vulnerability_artifact() {
+	if [ "$INFRA_ERROR_DETECTED" -ne 1 ]; then
+		return 1
+	fi
+	if has_any_vulnerability_report_artifact; then
+		return 1
+	fi
+	return 0
+}
+
 fail_reported_vulnerabilities_before_fallback_success() {
 	case "$PR_FINDINGS_DECISION" in
 	allow_baseline)
@@ -3890,8 +3973,11 @@ run_current_target_scan() {
 		if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then
 			return 1
 		fi
-		echo "Strix quick scan failed with a non-recoverable error." >&2
-		return 1
+		if ! is_infra_error_without_vulnerability_artifact; then
+			echo "Strix quick scan failed with a non-recoverable error." >&2
+			return 1
+		fi
+		echo "Provider infrastructure error left log-only severity markers and no Strix vulnerability report artifact; treating the failure as retryable and continuing to fallback models." >&2
 		;;
 	esac
 	if fail_unmapped_threshold_report; then
@@ -3967,8 +4053,11 @@ run_current_target_scan() {
 			if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then
 				return 1
 			fi
-			echo "Strix quick scan failed with a non-recoverable error." >&2
-			return 1
+			if ! is_infra_error_without_vulnerability_artifact; then
+				echo "Strix quick scan failed with a non-recoverable error." >&2
+				return 1
+			fi
+			echo "Provider infrastructure error left log-only severity markers and no Strix vulnerability report artifact; treating the failure as retryable and continuing to fallback models." >&2
 			;;
 		esac
 		if fail_unmapped_threshold_report; then

scripts/ci/strix_required_workflow_smoke.sh

@@ -149,6 +149,10 @@ assert_file_not_contains "$workflow_file" 'repository: ${{ github.repository }}'
 assert_file_not_contains "$workflow_file" 'bash "$TRUSTED_STRIX_GATE_TEST"' "Strix required path must not execute the full long-form gate harness"
 assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "Strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans"
 assert_file_contains "$gate_script" "STRIX_GITHUB_MODELS_KEY_FILE" "Strix gate supports GitHub Models fallback credentials for cross-provider fallback"
+assert_file_contains "$gate_script" "is_infra_error_without_vulnerability_artifact" "Strix gate classifies artifact-less provider crashes as retryable"
+assert_file_contains "$gate_script" "Provider infrastructure error left log-only severity markers and no Strix vulnerability report artifact; treating the failure as retryable and continuing to fallback models." "Strix gate continues to fallback models after artifact-less provider crashes"
+assert_file_contains "$gate_script" "capture_attempt_preexisting_report_run_dirs" "Strix gate scopes report failure signals to the current attempt"
+assert_file_contains "$workflow_file" "Provider outages without a vulnerability report are incomplete security evidence and are not converted to success" "Strix workflow preserves fail-closed exhaustion after fallback attempts"
 assert_file_contains "$gate_script" "STRIX_REPO_ROOT" "Strix gate consumes explicit target root"
 assert_file_contains "$gate_script" "STRIX_REPO_ROOT must reference a regular directory" "Strix gate rejects invalid or symlink target roots"
 assert_file_contains "$gate_script" "TARGET_PATH_IS_INTERNAL_PR_SCOPE" "Strix gate separates generated PR scopes from user paths"