test(engines): binary-gated smoke/contract tier across every engine adapter (#915)

.github/workflows/engine-smoke.yml

@@ -0,0 +1,184 @@
+name: Engine Smoke
+
+# Runs the shipped engine adapters against their REAL CLIs (#915).
+#
+# Three engines shipped with invocations that could not do any work — #913
+# (opencode's non-existent --non-interactive), #914 (codex's invented
+# app-server handshake), #1012 (kilocode's non-existent `run` subcommand) —
+# every one of them behind a green suite, because the adapter tests mock
+# subprocess.Popen and assert the adapter agrees with itself.
+#
+# Two legs, gated differently on purpose:
+#   contract — CLI exists, responds, still documents the adapter's entry point.
+#              No credentials, no model calls, seconds. Runs on every PR so it
+#              can be a required check for adapter changes.
+#   task     — one trivial task per engine driven to a terminal state with a
+#              real file written. Needs credentials, costs money, takes minutes.
+#              Scheduled and manual only.
+
+on:
+  pull_request:
+  schedule:
+    # Weekly — Sunday 4am UTC, an hour after the lifecycle run.
+    - cron: '0 4 * * 0'
+  workflow_dispatch:
+
+env:
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  contract:
+    # Always runs — a path-filtered required check reports "skipped" and blocks
+    # the merge forever, so the filtering happens inside the job instead.
+    name: Engine CLI Contract
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Detect engine adapter changes
+        id: changed
+        run: |
+          if [ "${{ github.event_name }}" != "pull_request" ]; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          base="origin/${{ github.base_ref }}"
+          git fetch -q origin "${{ github.base_ref }}"
+          if git diff --name-only "$base"...HEAD \
+              | grep -qE '^(codeframe/core/adapters/|tests/core/adapters/|\.github/workflows/engine-smoke\.yml)'; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "relevant=false" >> "$GITHUB_OUTPUT"
+            echo "No engine adapter changes — contract checks not required."
+          fi
+
+      - name: Set up Python
+        if: steps.changed.outputs.relevant == 'true'
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install uv
+        if: steps.changed.outputs.relevant == 'true'
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        if: steps.changed.outputs.relevant == 'true'
+        run: |
+          uv venv
+          uv sync --extra dev
+          uv pip install -e .
+
+      - name: Install engine CLIs
+        if: steps.changed.outputs.relevant == 'true'
+        # No `|| true` here. A failed install leaves the binary absent, the tier
+        # skips that engine, and pytest still exits 0 — a required check that is
+        # green while covering nothing, which is the exact failure mode this
+        # tier exists to end.
+        run: |
+          npm install -g \
+            @anthropic-ai/claude-code \
+            @openai/codex \
+            opencode-ai \
+            @kilocode/cli
+
+      - name: Verify every engine CLI is present
+        if: steps.changed.outputs.relevant == 'true'
+        run: |
+          missing=()
+          for b in claude codex opencode kilo; do
+            if command -v "$b" >/dev/null; then
+              echo "ok: $b -> $(command -v "$b")"
+            else
+              missing+=("$b")
+            fi
+          done
+          if [ ${#missing[@]} -ne 0 ]; then
+            echo "::error::engine CLIs missing: ${missing[*]}. The contract" \
+                 "checks for these would skip, leaving this required check" \
+                 "green while covering nothing."
+            exit 1
+          fi
+
+      - name: Run the contract leg
+        if: steps.changed.outputs.relevant == 'true'
+        # No CODEFRAME_ENGINE_SMOKE and no secrets: the task leg stays skipped.
+        run: |
+          uv run pytest tests/core/adapters/test_engine_smoke_tier.py \
+            -v --no-header -p no:randomly
+
+  task:
+    # Real credentials and real model calls — never on a pull request.
+    name: Engine Task Smoke
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    environment: staging
+    timeout-minutes: 45
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
+        with:
+          enable-cache: true
+
+      - name: Install dependencies
+        run: |
+          uv venv
+          uv sync --extra dev
+          uv pip install -e .
+
+      - name: Install engine CLIs
+        run: |
+          npm install -g \
+            @anthropic-ai/claude-code \
+            @openai/codex \
+            opencode-ai \
+            @kilocode/cli
+
+      - name: Verify every engine CLI is present
+        # Same reasoning as the contract job: a canary that silently stops
+        # canarying is worse than no canary.
+        run: |
+          missing=()
+          for b in claude codex opencode kilo; do
+            if command -v "$b" >/dev/null; then
+              echo "ok: $b -> $(command -v "$b")"
+            else
+              missing+=("$b")
+            fi
+          done
+          if [ ${#missing[@]} -ne 0 ]; then
+            echo "::error::engine CLIs missing: ${missing[*]}"
+            exit 1
+          fi
+
+      - name: Configure git
+        run: |
+          git config --global user.name "Engine Smoke"
+          git config --global user.email "engine-smoke@codeframe.test"
+
+      - name: Run the full tier
+        env:
+          CODEFRAME_ENGINE_SMOKE: '1'
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          uv run pytest tests/core/adapters/test_engine_smoke_tier.py \
+            tests/core/adapters/test_opencode_smoke_913.py \
+            tests/core/adapters/test_kilocode_smoke_1012.py \
+            -v --no-header -p no:randomly -rs