From b83eda81f8e94f2c8c9c3354249d5bebb8c48aa3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 5 Aug 2026 13:29:44 +0900 Subject: [PATCH] fix(strix): integrate hosted NVIDIA fallback on current trusted uv head --- .github/workflows/strix.yml | 12 +- CHANGELOG.md | 11 + .../strix-nvidia-nim-not-found-fallback.md | 76 +++++ scripts/ci/strix_quick_gate.sh | 24 +- scripts/ci/strix_required_workflow_smoke.sh | 5 + scripts/ci/test_strix_quick_gate.sh | 6 +- ...est_strix_nvidia_nim_not_found_fallback.py | 261 ++++++++++++++++++ 7 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 docs/doctoring/strix-nvidia-nim-not-found-fallback.md create mode 100644 tests/test_strix_nvidia_nim_not_found_fallback.py diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 7a39127d0..03ec23257 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -443,7 +443,7 @@ jobs: - name: Gate Strix secrets id: gate env: - STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b' || 'gpt-5.6-luna') }} + STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.6-luna') }} STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} STRIX_OPENAI_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} STRIX_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} @@ -453,7 +453,7 @@ jobs: TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} run: | strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]; then + if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]; then strix_model="gpt-5.6-luna" fi echo "strix_model=$strix_model" >> "$GITHUB_OUTPUT" @@ -497,7 +497,7 @@ jobs: exit 1 fi ;; - nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b) + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) if [ "$TARGET_REPOSITORY_PRIVATE" != "false" ]; then echo '::error::NVIDIA NIM hosted trial scans are limited to public repositories.' exit 1 @@ -763,7 +763,7 @@ jobs: openrouter/free | openrouter/openrouter/free) printf '%s' 'openrouter/free' > "$strix_llm_file" ;; - nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b) + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) printf '%s' "$strix_model" > "$strix_llm_file" ;; vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash) @@ -807,7 +807,7 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openrouter' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openrouter' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" @@ -861,7 +861,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..a607be5ca --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# Changelog + +All notable changes to the organization automation repository are documented in +this file. The format follows Keep a Changelog, and versioned releases follow +Semantic Versioning where the repository publishes a release. + +## [Unreleased] + +### Fixed + +- Made Strix treat only a single LiteLLM provider-error line containing NVIDIA NIM context and model-catalog 404 evidence as cross-model fallback evidence, rejecting cross-line signal assembly and provider-like target source literals; moved the public default to Nemotron 3 Super 120B and added a second NVIDIA hosted candidate before GitHub Models without neutralizing reported vulnerabilities. diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md new file mode 100644 index 000000000..70299ebdf --- /dev/null +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -0,0 +1,76 @@ +# Strix NVIDIA NIM model-catalog fallback: evidence and design record + +## Decision + +Strix treats an authenticated NVIDIA NIM model-catalog `404 Not Found` as +provider availability evidence, not as a target-application vulnerability. The +gate does not retry the same unavailable model. It proceeds to a distinct +reviewed NVIDIA hosted model and only then to the existing GitHub Models +candidates. + +Public-repository scans now default to +`nvidia/nemotron-3-super-120b-a12b`. The first fallback is +`nvidia/llama-3.3-nemotron-super-49b-v1.5`. Private repositories retain the +contracted provider because NVIDIA hosted trial inputs are restricted to public +repositories by the central workflow. + +## Trust boundary + +The NVIDIA classifier accepts only a single bounded log line that contains all +three signals: a LiteLLM `NotFoundError`, NVIDIA NIM provider context, and +model-catalog not-found evidence. It does not assemble provider and `404` +signals from different lines. A bare application `404`, route miss, database +lookup miss, provider-like source literal, or other target-controlled output is +not enough to enter model fallback. + +This same-line rule matters because scanner stdout can include text derived from +the repository under review. Requiring the trusted LiteLLM exception marker and +all provider-availability evidence on one line prevents repository content from +combining with an unrelated application `404` to spoof infrastructure fallback. +Provider-side failure also remains a fail-closed incomplete scan until a distinct +fallback produces complete evidence. + +The outer workflow may classify exhausted provider infrastructure as neutral only +when the run log contains no vulnerability signal. Any reported severity or +non-zero vulnerability count remains blocking. Scanner reports and attempt logs +remain available as artifacts. + +## Verification contract + +Regression evidence proves that: + +1. the exact LiteLLM `Nvidia_nimException` 404 observed in required CI is + recognized; +2. an ordinary application 404 is not recognized; +3. provider context and 404 evidence on different lines are not recognized; +4. a provider-like source literal on one line without LiteLLM `NotFoundError` + context is not recognized; +5. model-catalog 404s enter cross-model fallback but never same-model retry; +6. the primary and first fallback are current NVIDIA hosted models; +7. GitHub Models remain later cross-provider fallbacks; +8. vulnerability signals prevent neutral infrastructure classification; and +9. the required-workflow smoke contract pins these properties. + +## Limitations + +Hosted model catalogs may change independently of this repository. A model-card +page or supported self-hosted NIM container does not guarantee indefinite hosted +trial availability. The ordered model plan must therefore be reviewed against +current NVIDIA documentation whenever a provider returns a catalog 404. This +change does not treat arbitrary provider errors as success and does not weaken +Strix severity, changed-file attribution, or independent approval requirements. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC +9110). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +NVIDIA Corporation. (2025). *Llama-3.3-Nemotron-Super-49B-v1.5* [Model card]. +NVIDIA NIM. https://build.nvidia.com/nvidia/llama-3_3-nemotron-super-49b-v1_5/modelcard + +NVIDIA Corporation. (2026a). *NVIDIA-Nemotron-3-Super-120B-A12B* [Model +card]. NVIDIA NIM. +https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b/modelcard + +NVIDIA Corporation. (2026b). *Configuration reference*. NVIDIA AI-Q Blueprint. +https://docs.nvidia.com/aiq-blueprint/2.2.0-rc1/customization/configuration-reference.html diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3b001a921..c318f788f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2641,6 +2641,20 @@ is_llm_service_unavailable_error() { return 1 } +is_nvidia_nim_not_found_error() { + # Classify only one bounded LiteLLM provider-error line that also + # carries NVIDIA NIM context and model-catalog not-found evidence. + # Cross-line signal assembly and provider-like target source text + # remain non-retryable so application output cannot spoof fallback. + if grep -Ei 'litellm(\.exceptions)?\.NotFoundError' "$STRIX_LOG" | + grep -Ei '(Nvidia_nimException|nvidia[_ -]?nim|integrate\.api\.nvidia\.com)' | + grep -Eiq '(Error code:[[:space:]]*404|(^|[^0-9])404([^0-9]|$)|model[^[:alnum:]]+not found)'; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). ## Four error families qualify: @@ -2909,7 +2923,7 @@ is_midstream_fallback_error() { # (httpx, httpcore, requests). Used for generic transport failures where # library names alone are insufficient to prove the timeout/connection error # originated from an LLM provider rather than the target application. -LLM_PROVIDER_ONLY_REGEX='(litellm|openai|anthropic|VertexAI|Vertex_ai|vertex\.ai|google\.cloud|GitHub Models|models\.github\.ai|github_models)' +LLM_PROVIDER_ONLY_REGEX='(litellm|openai|anthropic|VertexAI|Vertex_ai|vertex\.ai|google\.cloud|Nvidia_nimException|nvidia_nim|integrate\.api\.nvidia\.com|GitHub Models|models\.github\.ai|github_models)' is_llm_token_limit_error() { if grep -Eiq '(tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|(^|[^0-9])413([^0-9]|$))' "$STRIX_LOG" && @@ -2953,6 +2967,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_nvidia_nim_not_found_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3799,6 +3817,10 @@ is_model_retryable_error() { return 0 fi + if is_nvidia_nim_not_found_error; then + return 0 + fi + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then return 0 fi diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 57df964a1..8cd6dddad 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -155,6 +155,11 @@ assert_file_contains "$gate_script" "TARGET_PATH_IS_INTERNAL_PR_SCOPE" "Strix ga assert_file_contains "$gate_script" "NPM_CONFIG_IGNORE_SCRIPTS" "Strix gate disables npm lifecycle scripts" assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardened" "Full Strix harness remains available outside the required path" +assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" +assert_file_contains "$workflow_file" "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat" "Strix tries another NVIDIA hosted model before GitHub Models" +assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" +assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" + if [ "$failures" -ne 0 ]; then echo "Strix required workflow smoke test failed with $failures failure(s)." >&2 exit 1 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b4d585b9e..4e317e535 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -289,8 +289,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" "STRIX_TOTAL_TIMEOUT_SECONDS:" "strix workflow must not expose total timeout env names in GitHub logs" assert_file_not_contains "$workflow_file" "STRIX_PR_SCOPE_MAX_FILES_PER_BATCH" "strix workflow must not split Strix PR evidence into separate scanner runs" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM == 'vertex_ai/gemini-3.1-pro-preview-customtools' && 'vertex_ai/gemini-2.5-flash'" "strix workflow must not quarantine the approved Vertex preview model after organization secret visibility is fixed" - assert_file_contains "$workflow_file" "steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b' || 'gpt-5.6-luna'" "strix workflow defaults public scans to NVIDIA NIM and keeps private scans on the contracted provider" - assert_file_contains "$workflow_file" 'if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]' "strix workflow falls back to the contracted provider when the NVIDIA secret is absent" + assert_file_contains "$workflow_file" "steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.6-luna'" "strix workflow defaults public scans to NVIDIA NIM and keeps private scans on the contracted provider" + assert_file_contains "$workflow_file" 'if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]' "strix workflow falls back to the contracted provider when the NVIDIA secret is absent" assert_file_contains "$workflow_file" 'STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}' "strix workflow propagates the gate-selected fallback model to the scanner" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM ||" "strix workflow must not let the legacy STRIX_LLM secret override PR defaults" assert_file_contains "$workflow_file" "STRIX_LLM must select NVIDIA NIM Nemotron, GitHub Models openai/gpt-5 or newer, direct OpenAI GPT-5.4 or newer, OpenRouter openrouter/free, or an approved organization Vertex AI model" "strix workflow rejects unsupported model inputs" @@ -348,7 +348,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "github_models/openai/o3 github_models/openai/gpt-5-chat" "strix workflow keeps GitHub Models fallback on tool-capable OpenAI models without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives direct-OpenAI scans GitHub Models fallbacks so provider quota outages degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py new file mode 100644 index 000000000..a48f3092d --- /dev/null +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -0,0 +1,261 @@ +"""Regression contract for NVIDIA NIM model retirement and hosted 404 fallback. + +The central Strix workflow must not turn a provider-side model-catalog 404 into a +security finding or retry the same unavailable model. It must move to another +approved free NVIDIA NIM candidate before using the existing GitHub Models +fallbacks, while ordinary application 404 output remains non-retryable. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" +DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" +FREE_NVIDIA_FALLBACK = ( + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" +) +RETIRED_PRIMARY_MODEL = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace. + + The relevant Strix classifier functions contain no nested top-level function + declarations. Requiring a brace on a line by itself keeps extraction bounded + and makes source-shape drift fail the test instead of silently selecting the + wrong shell code. + """ + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_nvidia_not_found(log_text: str) -> bool: + """Execute the production classifier against a bounded synthetic log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block( + gate_source, + "is_nvidia_nim_not_found_error", + ) + with tempfile.TemporaryDirectory(prefix="strix-nvidia-404-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_nvidia_nim_not_found_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" + + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + """Execute the outer workflow's backend-neutralization condition.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern( + workflow, + "backend_unavailable_signal", + ) + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + with tempfile.TemporaryDirectory(prefix="strix-workflow-404-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return backend.returncode == 0 and vulnerability.returncode == 1 + + +class StrixNvidiaNotFoundFallbackTests(unittest.TestCase): + """Protect provider-scoped 404 fallback without weakening security gates.""" + + def test_nvidia_hosted_model_404_is_retryable_provider_evidence(self) -> None: + """Recognize the exact LiteLLM/NVIDIA 404 observed in required CI.""" + + log = ( + "litellm.exceptions.NotFoundError: Nvidia_nimException - " + "Error code: 404\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_nvidia_not_found(log)) + + def test_application_404_without_nvidia_context_is_not_retryable(self) -> None: + """Do not let target-application HTTP 404 text bypass security evidence.""" + + log = "GET /api/project_record/unknown 404\nNotFoundError: record missing\n" + self.assertFalse(_classifies_as_nvidia_not_found(log)) + + def test_provider_and_404_signals_must_share_one_log_line(self) -> None: + """Reject cross-line signal assembly from untrusted scan-target output.""" + + log = ( + "source literal: Nvidia_nimException\n" + "GET /api/project_record/unknown Error code: 404\n" + ) + self.assertFalse(_classifies_as_nvidia_not_found(log)) + + def test_provider_literal_without_litellm_error_is_not_retryable(self) -> None: + """Reject source text that imitates an NVIDIA provider error line.""" + + log = "source literal: Nvidia_nimException Error code: 404\n" + self.assertFalse(_classifies_as_nvidia_not_found(log)) + + def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> None: + """Wire the classifier only into infrastructure and model fallback.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block( + gate_source, + "has_detected_infrastructure_error", + ) + retryable = _function_block(gate_source, "is_model_retryable_error") + same_model_retry = _function_block( + gate_source, + "is_transient_same_model_retry_error", + ) + + self.assertIn("is_nvidia_nim_not_found_error", infrastructure) + self.assertIn("is_nvidia_nim_not_found_error", retryable) + self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) + + def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: + """Prefer a documented hosted NIM and another NIM before GitHub.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + default_expression = ( + "steps.target_visibility.outputs.is_private == 'false' && " + f"'{DEFAULT_NVIDIA_MODEL}' || 'gpt-5.6-luna'" + ) + self.assertIn(default_expression, workflow) + self.assertIn( + f'[ "$strix_model" = "{DEFAULT_NVIDIA_MODEL}" ] ' + '&& [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]', + workflow, + ) + self.assertIn( + "steps.gate.outputs.provider_mode == 'nvidia_nim' && " + f"'{FREE_NVIDIA_FALLBACK} github_models/openai/o3 " + "github_models/openai/gpt-5-chat'", + workflow, + ) + + default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] + default_gate = default_gate.split( + "- name: Prepare LLM API key input file", + maxsplit=1, + )[0] + self.assertNotIn(RETIRED_PRIMARY_MODEL, default_gate) + + def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: + """Reject provider-like target text in the outer neutralization gate.""" + + self.assertFalse( + _workflow_neutralizes( + "source literal: Nvidia_nimException Error code: 404\n" + ) + ) + self.assertTrue( + _workflow_neutralizes( + "litellm.exceptions.NotFoundError: Nvidia_nimException - " + "Error code: 404\nVulnerabilities 0\n" + ) + ) + + def test_outer_workflow_rejects_cross_line_signal_assembly(self) -> None: + """Require exception, provider, and 404 evidence on one physical line.""" + + self.assertFalse( + _workflow_neutralizes( + "litellm.exceptions.NotFoundError: provider unavailable\n" + "Nvidia_nimException Error code: 404\n" + ) + ) + + def test_outer_workflow_rejects_nvidia_404_without_litellm_context(self) -> None: + """Require LiteLLM NotFoundError context, not just NVIDIA + 404.""" + + self.assertFalse( + _workflow_neutralizes( + "Nvidia_nimException Error code: 404\nVulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: + """Keep a real vulnerability signal blocking despite provider failure.""" + + self.assertFalse( + _workflow_neutralizes( + "litellm.exceptions.NotFoundError: Nvidia_nimException - " + "Error code: 404\nVulnerabilities 1\n" + ) + ) + + def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: + """Retain the static fail-closed vulnerability evidence contract.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("Nvidia_nimException", workflow) + self.assertIn("Error code:[[:space:]]*404", workflow) + self.assertIn("reported_vulnerability_signal", workflow) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + '! grep -Eiq "$reported_vulnerability_signal"', + workflow, + ) + + +if __name__ == "__main__": + unittest.main()