diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3c12948..e5f8ebe 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -17,8 +17,8 @@ jobs: with: go-version-file: go.mod - - name: Build harness CLI - run: make cli + - name: Install bats + run: sudo apt-get install -y bats - name: Install openshell run: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh @@ -31,8 +31,8 @@ jobs: done openshell gateway list - - name: Run integration tests (ci mode — no credentials) - run: ./test/test-flow.sh local --ci + - name: Run CI local + run: make ci-local - name: Export gateway logs if: always() @@ -56,8 +56,8 @@ jobs: with: go-version-file: go.mod - - name: Build harness CLI - run: make cli + - name: Install bats + run: sudo apt-get install -y bats - name: Install openshell run: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh @@ -69,11 +69,8 @@ jobs: - name: Install helm uses: azure/setup-helm@v4 - - name: Deploy gateway to kind - run: ./harness deploy kind - - - name: Run integration tests (ci mode — no credentials) - run: ./test/test-flow.sh kind --ci + - name: Run CI kind + run: make ci-kind - name: Export logs if: always() diff --git a/Makefile b/Makefile index b9692c8..a5d4a55 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,19 @@ ## OpenShell Harness — build, push, and test ## -## Usage: -## make sandbox # build + push sandbox image (multi-arch) -## make launcher # build + push launcher image -## make test # build images + run full tests on both platforms -## make test-podman # build + test podman only -## make test-ocp # build + test OCP only +## CI targets (no credentials, GHA-safe): +## make ci # unit + bats + lint (~2min) +## make ci-local # ci + local gateway integration +## make ci-kind # ci + kind (self-contained cluster) +## +## Developer targets (credentials available): +## make dev-test-local # pre-commit: unit + bats + local full + ci +## make dev-test-kind # kind: self-contained lifecycle +## make dev-test-remote # OCP: needs KUBECONFIG +## make dev-test-all # all of the above +## +## Images: +## make sandbox # build + push sandbox (multi-arch) +## make launcher # build + push launcher REGISTRY ?= ghcr.io/robbycochran/harness-openshell DEV_REGISTRY ?= quay.io/rcochran/openshell @@ -18,10 +26,9 @@ DEV_SANDBOX_IMAGE := $(DEV_REGISTRY):$(DEV_TAG)-sandbox DEV_LAUNCHER_IMAGE := $(DEV_REGISTRY):$(DEV_TAG)-launcher .PHONY: cli sandbox push-sandbox cli-launcher launcher push-launcher \ - vet lint test-unit test test-local test-kind test-ocp test-all validate \ - validate-local validate-local-ci validate-kind validate-kind-ci \ - validate-ocp validate-ocp-ci \ - dev-sandbox dev-launcher validate-dev clean help + vet lint ci ci-local ci-kind \ + dev-test-local dev-test-kind dev-test-remote dev-test-all \ + dev-sandbox dev-launcher clean help ## ── CLI ────────────────────────────────────────────────────────────── @@ -70,155 +77,69 @@ lint: $(MAKE) vet; \ fi -## ── Test targets ───────────────────────────────────────────────────── +## ── CI targets (no credentials, GHA-safe) ──────────────────────────── -## Unit tests only (no live gateway, fast) -test-unit: +## Unit tests + bats + lint (fast, ~2min, no gateway needed) +ci: vet CGO_ENABLED=0 go test ./... cd sandbox/launcher && go test ./... bats test/preflight.bats -## Both platforms (full lifecycle, rebuilds images) -test: cli sandbox push-launcher - ./test/test-flow.sh all --full - -## Local gateway (full lifecycle) -test-local: cli - ./test/test-flow.sh local --full - -## kind cluster (full lifecycle — requires: kind create cluster --name openshell) -test-kind: cli - ./test/test-flow.sh kind --full +## CI + local gateway integration (ci mode, no credentials) +ci-local: cli ci + ./test/test-flow.sh local --ci -## OCP only (full lifecycle) -test-ocp: cli sandbox push-launcher - ./test/test-flow.sh ocp --full +## CI + kind gateway integration (self-contained, isolated kubeconfig) +ci-kind: cli ci + ./test/kind-lifecycle.sh --ci -## All combinations: local + kind + ocp -test-all: cli sandbox push-launcher - ./test/test-flow.sh all --full +## ── Developer targets (credentials available) ──────────────────────── -## Full validation: unit tests + bats + integration (local + OCP) -## Run this before every commit. -validate: cli sandbox push-launcher - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... +## Pre-commit local: unit + bats + local full + local CI +## Requires: openshell gateway running locally, provider credentials. +dev-test-local: cli ci @echo "" - @echo "=== Bats ===" - bats test/preflight.bats - @echo "" - @echo "=== Integration: local ===" + @echo "=== Integration: local (full) ===" ./test/test-flow.sh local --full @echo "" - @echo "=== Integration: OCP ===" - ./test/test-flow.sh ocp --full - -## Dev validation: unit tests + bats + build images + full integration matrix. -## Builds sandbox + launcher to DEV_REGISTRY (quay.io/rcochran/openshell), runs every flow. -## Requires: openshell gateway running locally, OCP cluster via KUBECONFIG. -dev-sandbox: - docker buildx build --platform linux/amd64,linux/arm64 -t $(DEV_SANDBOX_IMAGE) sandbox/ --push - @echo "Built and pushed: $(DEV_SANDBOX_IMAGE)" - -dev-launcher: cli-launcher - docker build --platform $(PLATFORM) -t $(DEV_LAUNCHER_IMAGE) sandbox/launcher/ - docker push $(DEV_LAUNCHER_IMAGE) - @echo "Built and pushed: $(DEV_LAUNCHER_IMAGE)" + @echo "=== Integration: local (ci) ===" + ./test/test-flow.sh local --ci -validate-dev: cli dev-sandbox dev-launcher - @echo "=== Images ===" - @echo " SANDBOX_IMAGE: $(DEV_SANDBOX_IMAGE)" - @echo " LAUNCHER_IMAGE: $(DEV_LAUNCHER_IMAGE)" - @echo "" - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... +## Kind: unit + bats + kind full (self-contained lifecycle) +## Creates/destroys its own kind cluster. Never touches your OCP kubectl context. +## Builds dev sandbox image to quay.io (kind can't pull private ghcr.io). +## Use KEEP=1 to keep the cluster after tests (for debugging). +dev-test-kind: cli ci dev-sandbox @echo "" - @echo "=== Bats ===" - bats test/preflight.bats - @echo "" - @echo "=== Integration: local (quick) ===" - SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) ./test/test-flow.sh local - @echo "" - @echo "=== Integration: local (full) ===" - SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) ./test/test-flow.sh local --full - @echo "" - @echo "=== Integration: local CI profile (no providers) ===" - ./test/test-flow.sh local --full --no-providers --profile=ci - @echo "" - @echo "=== Integration: OCP (quick) ===" - SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) LAUNCHER_IMAGE=$(DEV_LAUNCHER_IMAGE) ./test/test-flow.sh ocp + SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) SANDBOX_PULL_SECRET=quay-pull ./test/kind-lifecycle.sh $(if $(KEEP),--keep) + +## Remote (OCP): unit + bats + OCP full + OCP CI +## Requires: KUBECONFIG set, provider credentials. +## Builds dev images to quay.io (OCP can't pull private ghcr.io). +dev-test-remote: cli ci dev-sandbox dev-launcher + @test -n "$${KUBECONFIG}" || { echo "ERROR: Set KUBECONFIG for OCP (e.g. export KUBECONFIG=infracluster/kubeconfig)"; exit 1; } @echo "" @echo "=== Integration: OCP (full) ===" SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) LAUNCHER_IMAGE=$(DEV_LAUNCHER_IMAGE) ./test/test-flow.sh ocp --full - -## Default validation: unit tests + bats + full integration with user credentials. -## Requires: openshell gateway running locally (brew services start openshell), -## JIRA_API_TOKEN, gcloud ADC, gws auth login. -validate-local: cli - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... - @echo "" - @echo "=== Bats ===" - bats test/preflight.bats @echo "" - @echo "=== Integration: local gateway, default mode ===" - ./test/test-flow.sh local --full + @echo "=== Integration: OCP (ci) ===" + LAUNCHER_IMAGE=$(DEV_LAUNCHER_IMAGE) ./test/test-flow.sh ocp --ci -## CI validation: unit tests + integration without credentials (local gateway). -## Requires: openshell gateway running locally only. -validate-local-ci: cli - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... - @echo "" - @echo "=== Integration: local gateway, ci mode ===" - ./test/test-flow.sh local --ci +## All: local + kind + remote +dev-test-all: dev-test-local dev-test-kind dev-test-remote -## Default validation on kind: unit tests + full integration with user credentials. -## Requires: kind create cluster --name openshell. Builds dev sandbox image to quay.io/rcochran. -validate-kind: cli dev-sandbox - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... - @echo "" - @echo "=== Integration: kind gateway, default mode ===" - @kubectl create namespace openshell --dry-run=client -o yaml 2>/dev/null | kubectl apply -f - 2>/dev/null || true - @kubectl create secret docker-registry quay-pull \ - --docker-server=quay.io --docker-username=rcochran \ - --docker-password="$$(python3 -c "import json,base64,pathlib; d=json.loads(pathlib.Path('$$HOME/.docker/config.json').read_text()); print(base64.b64decode(d['auths']['quay.io']['auth']).decode().split(':',1)[1])")" \ - -n openshell --dry-run=client -o yaml 2>/dev/null | kubectl apply -f - 2>/dev/null || true - SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) SANDBOX_PULL_SECRET=quay-pull ./test/test-flow.sh kind --full - -## CI validation on kind: unit tests + integration without credentials. -## Requires: kind create cluster --name openshell -validate-kind-ci: cli - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... - @echo "" - @echo "=== Integration: kind gateway, ci mode ===" - ./test/test-flow.sh kind --ci +## ── Dev image builds ───────────────────────────────────────────────── -## OCP validation with credentials (default mode). Requires: KUBECONFIG, all provider creds. -validate-ocp: cli dev-launcher dev-sandbox - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... - @echo "" - @echo "=== Integration: OCP gateway, default mode ===" - SANDBOX_IMAGE=$(DEV_SANDBOX_IMAGE) LAUNCHER_IMAGE=$(DEV_LAUNCHER_IMAGE) ./test/test-flow.sh ocp --full +## Build dev sandbox image to quay.io (multi-arch) +dev-sandbox: + docker buildx build --platform linux/amd64,linux/arm64 -t $(DEV_SANDBOX_IMAGE) sandbox/ --push + @echo "Built and pushed: $(DEV_SANDBOX_IMAGE)" -## OCP validation without credentials (ci mode). Requires: KUBECONFIG, launcher image pushed. -validate-ocp-ci: cli dev-launcher - @echo "=== Unit tests ===" - CGO_ENABLED=0 go test ./... - cd sandbox/launcher && go test ./... - @echo "" - @echo "=== Integration: OCP gateway, ci mode ===" - LAUNCHER_IMAGE=$(DEV_LAUNCHER_IMAGE) ./test/test-flow.sh ocp --ci --full +## Build dev launcher image to quay.io +dev-launcher: cli-launcher + docker build --platform $(PLATFORM) -t $(DEV_LAUNCHER_IMAGE) sandbox/launcher/ + docker push $(DEV_LAUNCHER_IMAGE) + @echo "Built and pushed: $(DEV_LAUNCHER_IMAGE)" ## ── Convenience targets ─────────────────────────────────────────────── diff --git a/agents/ci.yaml b/agents/ci.yaml new file mode 100644 index 0000000..8908fb3 --- /dev/null +++ b/agents/ci.yaml @@ -0,0 +1,9 @@ +# CI integration test profile. +# +# Minimal sandbox with no providers and no credentials. +# Uses the public community base image (no registry auth needed). + +name: ci-test +image: ghcr.io/nvidia/openshell-community/sandboxes/base:latest +entrypoint: bash +providers: [] diff --git a/agents/default.yaml b/agents/default.yaml new file mode 100644 index 0000000..0a21ab5 --- /dev/null +++ b/agents/default.yaml @@ -0,0 +1,26 @@ +# Default agent configuration. +# +# Usage: +# harness up # uses agents/default.yaml +# harness up --agent research # uses agents/research.yaml + +name: agent +image: ghcr.io/robbycochran/harness-openshell:sandbox +entrypoint: claude --bare +tty: true + +providers: + - type: github + - type: vertex-local + - type: atlassian + config: + JIRA_URL: ${JIRA_URL} + JIRA_USERNAME: ${JIRA_USERNAME} + - type: gws + +# Non-secret env vars injected into sandbox. +# Proxy config: gateway routes inference through its local proxy. +env: + ANTHROPIC_BASE_URL: https://inference.local + ANTHROPIC_API_KEY: sk-ant-openshell-proxy-managed + CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS: "1" diff --git a/internal/agent/agent.go b/internal/agent/agent.go index e047d62..6baf5bf 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -16,14 +16,15 @@ type ProviderRef struct { } type AgentConfig struct { - Name string `yaml:"name"` - Providers []ProviderRef `yaml:"providers"` - Task string `yaml:"task,omitempty"` - Entrypoint string `yaml:"entrypoint,omitempty"` - TTY *bool `yaml:"tty,omitempty"` - Policy string `yaml:"policy,omitempty"` - Image string `yaml:"image,omitempty"` - Include []string `yaml:"include,omitempty"` + Name string `yaml:"name"` + Providers []ProviderRef `yaml:"providers"` + Env map[string]string `yaml:"env,omitempty"` + Task string `yaml:"task,omitempty"` + Entrypoint string `yaml:"entrypoint,omitempty"` + TTY *bool `yaml:"tty,omitempty"` + Policy string `yaml:"policy,omitempty"` + Image string `yaml:"image,omitempty"` + Include []string `yaml:"include,omitempty"` } func (c *AgentConfig) NoTTY() bool { @@ -74,6 +75,9 @@ func Parse(data []byte) (*AgentConfig, error) { func (c *AgentConfig) BuildEnvSh() string { env := make(map[string]string) + for k, v := range c.Env { + env[k] = v + } for _, p := range c.Providers { for k, v := range p.Config { env[k] = v diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index f3844cd..01666a9 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -169,6 +169,70 @@ func TestBuildEnvSh_Empty(t *testing.T) { } } +func TestBuildEnvSh_TopLevelEnv(t *testing.T) { + cfg := &AgentConfig{ + Env: map[string]string{ + "ANTHROPIC_BASE_URL": "https://inference.local", + "ANTHROPIC_API_KEY": "sk-proxy", + }, + Providers: []ProviderRef{ + {Type: "atlassian", Config: map[string]string{"JIRA_URL": "https://jira.example.com"}}, + }, + } + env := cfg.BuildEnvSh() + if !strings.Contains(env, `export ANTHROPIC_BASE_URL="https://inference.local"`) { + t.Errorf("missing top-level env var in:\n%s", env) + } + if !strings.Contains(env, `export JIRA_URL="https://jira.example.com"`) { + t.Errorf("missing provider config var in:\n%s", env) + } +} + +func TestBuildEnvSh_ProviderOverridesTopLevel(t *testing.T) { + cfg := &AgentConfig{ + Env: map[string]string{"FOO": "from-top"}, + Providers: []ProviderRef{{Type: "test", Config: map[string]string{"FOO": "from-provider"}}}, + } + env := cfg.BuildEnvSh() + if !strings.Contains(env, `"from-provider"`) { + t.Errorf("provider config should override top-level env:\n%s", env) + } +} + +func TestParseFile_AgentYAML(t *testing.T) { + dir := t.TempDir() + yamlContent := `name: test-agent +image: ghcr.io/test:latest +entrypoint: claude --bare +tty: true +providers: + - type: github + - type: atlassian + config: + JIRA_URL: https://jira.example.com +env: + ANTHROPIC_BASE_URL: https://inference.local +` + os.WriteFile(filepath.Join(dir, "agent.yaml"), []byte(yamlContent), 0o644) + + cfg, err := ParseFile(filepath.Join(dir, "agent.yaml")) + if err != nil { + t.Fatalf("ParseFile: %v", err) + } + if cfg.Name != "test-agent" { + t.Errorf("Name = %q", cfg.Name) + } + if cfg.Image != "ghcr.io/test:latest" { + t.Errorf("Image = %q", cfg.Image) + } + if cfg.Env["ANTHROPIC_BASE_URL"] != "https://inference.local" { + t.Errorf("Env ANTHROPIC_BASE_URL = %q", cfg.Env["ANTHROPIC_BASE_URL"]) + } + if len(cfg.Providers) != 2 { + t.Errorf("Providers = %d, want 2", len(cfg.Providers)) + } +} + func TestBuildEnvSh_Sorted(t *testing.T) { cfg := &AgentConfig{ Providers: []ProviderRef{ diff --git a/test/kind-lifecycle.sh b/test/kind-lifecycle.sh new file mode 100755 index 0000000..791d6c7 --- /dev/null +++ b/test/kind-lifecycle.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Self-contained kind cluster lifecycle for integration testing. +# +# Creates a kind cluster with an isolated kubeconfig (never touches your +# OCP/cloud kubectl context), runs the test flow, then tears down. +# +# Usage: +# ./test/kind-lifecycle.sh # default mode (full, with creds) +# ./test/kind-lifecycle.sh --ci # ci mode (no creds) +# ./test/kind-lifecycle.sh --keep # don't delete cluster after tests +# +# Works alongside any existing KUBECONFIG. The cluster gets its own temp +# kubeconfig file that is cleaned up on exit. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +CLUSTER_NAME="openshell-test" +KIND_KUBECONFIG="" +KEEP_CLUSTER=false +TEST_ARGS=("kind" "--full") + +for arg in "$@"; do + case "$arg" in + --ci) TEST_ARGS+=("--ci") ;; + --keep) KEEP_CLUSTER=true ;; + *) TEST_ARGS+=("$arg") ;; + esac +done + +cleanup() { + local rc=$? + if $KEEP_CLUSTER; then + echo "" + echo "Cluster kept: $CLUSTER_NAME" + echo " KUBECONFIG=$KIND_KUBECONFIG kubectl get nodes" + echo " kind delete cluster --name $CLUSTER_NAME" + else + echo "" + echo "=== Deleting kind cluster ===" + kind delete cluster --name "$CLUSTER_NAME" 2>/dev/null || true + rm -f "$KIND_KUBECONFIG" + fi + exit $rc +} +trap cleanup EXIT + +# ── Create cluster with isolated kubeconfig ───────────────────────── + +KIND_KUBECONFIG=$(mktemp /tmp/kind-${CLUSTER_NAME}-XXXXXX.kubeconfig) +export KUBECONFIG="$KIND_KUBECONFIG" + +echo "=== kind cluster lifecycle ===" +echo " Cluster: $CLUSTER_NAME" +echo " KUBECONFIG: $KIND_KUBECONFIG" +echo "" + +if kind get clusters 2>/dev/null | grep -q "^${CLUSTER_NAME}$"; then + echo "=== Reusing existing cluster ===" + kind export kubeconfig --name "$CLUSTER_NAME" --kubeconfig "$KIND_KUBECONFIG" +else + echo "=== Creating kind cluster ===" + kind create cluster --name "$CLUSTER_NAME" --kubeconfig "$KIND_KUBECONFIG" +fi +echo "" + +# ── Pre-test setup ────────────────────────────────────────────────── + +kubectl create namespace openshell --dry-run=client -o yaml | kubectl apply -f - 2>/dev/null || true + +# Set up pull secret for dev images if docker credentials exist +if [[ -f "$HOME/.docker/config.json" ]]; then + QUAY_PASS=$(python3 -c " +import json, base64, pathlib, sys +try: + d = json.loads(pathlib.Path('$HOME/.docker/config.json').read_text()) + print(base64.b64decode(d['auths']['quay.io']['auth']).decode().split(':',1)[1]) +except Exception: + sys.exit(1) +" 2>/dev/null) || true + + if [[ -n "${QUAY_PASS:-}" ]]; then + kubectl create secret docker-registry quay-pull \ + --docker-server=quay.io --docker-username=rcochran \ + --docker-password="$QUAY_PASS" \ + -n openshell --dry-run=client -o yaml | kubectl apply -f - 2>/dev/null || true + fi +fi + +# ── Run tests ─────────────────────────────────────────────────────── + +echo "=== Running: test-flow.sh ${TEST_ARGS[*]} ===" +echo "" +"$SCRIPT_DIR/test/test-flow.sh" "${TEST_ARGS[@]}" diff --git a/test/preflight.bats b/test/preflight.bats index 7c481f2..0453724 100644 --- a/test/preflight.bats +++ b/test/preflight.bats @@ -493,6 +493,10 @@ EOF # ── CLI detection tests ────────────────────────────────────────────── @test "cli: missing openshell shows error" { + # Skip if openshell is installed at a system path we can't exclude + if [ -x /usr/bin/openshell ] || [ -x /usr/local/bin/openshell ]; then + skip "openshell installed at system path -- can't simulate missing CLI" + fi write_providers_toml <<'EOF' [[providers]] name = "test" diff --git a/test/test-flow.sh b/test/test-flow.sh index 192120a..52b28b2 100755 --- a/test/test-flow.sh +++ b/test/test-flow.sh @@ -121,8 +121,9 @@ check_providers() { sandbox_wait() { # Poll for sandbox Ready, failing fast on k8s bad states (ImagePullBackOff, etc.) + # 60 iterations * 2s = 120s timeout (kind needs extra time for cold image pulls) local name="$1" - for i in $(seq 1 30); do + for i in $(seq 1 60); do local phase phase=$("$CLI" sandbox list 2>/dev/null | strip_ansi | awk -v n="$name" '$1==n {print $NF}') [[ "$phase" == "Ready" ]] && return 0