diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..d0c1d7b --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,49 @@ +name: Integration + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + local: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build harness CLI + run: make cli + + - name: Install openshell + run: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh + + - name: Wait for gateway + run: | + for i in $(seq 1 30); do + openshell inference get &>/dev/null && break + sleep 1 + done + openshell gateway list + + - name: Run integration tests + run: ./test/test-flow.sh local --full --no-providers --profile=ci + + - name: Export gateway logs + if: always() + run: | + mkdir -p /tmp/gateway-logs + journalctl --user -u openshell-gateway --no-pager > /tmp/gateway-logs/gateway.log 2>/dev/null || true + openshell status > /tmp/gateway-logs/status.txt 2>&1 || true + openshell gateway list > /tmp/gateway-logs/gateways.txt 2>&1 || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: gateway-logs-local + path: /tmp/gateway-logs/ diff --git a/Makefile b/Makefile index 7c5aa83..d6319ad 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SANDBOX_IMAGE := $(REGISTRY):sandbox LAUNCHER_IMAGE := $(REGISTRY):launcher .PHONY: cli sandbox push-sandbox cli-launcher launcher push-launcher \ - vet lint test-unit test test-podman test-ocp test-all validate clean help + vet lint test-unit test test-local test-ocp test-all validate clean help ## ── CLI ────────────────────────────────────────────────────────────── @@ -75,9 +75,9 @@ test-unit: test: cli sandbox push-launcher ./test/test-flow.sh all --full -## Podman only (full lifecycle) -test-podman: cli sandbox push-launcher - ./test/test-flow.sh podman --full +## Local gateway (full lifecycle) +test-local: cli + ./test/test-flow.sh local --full ## OCP only (full lifecycle) test-ocp: cli sandbox push-launcher @@ -87,7 +87,7 @@ test-ocp: cli sandbox push-launcher test-all: cli sandbox push-launcher ./test/test-flow.sh all --full -## Full validation: unit tests + bats + integration (podman + OCP) +## Full validation: unit tests + bats + integration (local + OCP) ## Run this before every commit. validate: cli sandbox push-launcher @echo "=== Unit tests ===" @@ -97,8 +97,8 @@ validate: cli sandbox push-launcher @echo "=== Bats ===" bats test/preflight.bats @echo "" - @echo "=== Integration: podman ===" - ./test/test-flow.sh podman --full + @echo "=== Integration: local ===" + ./test/test-flow.sh local --full @echo "" @echo "=== Integration: OCP ===" ./test/test-flow.sh ocp --full diff --git a/cmd/deploy.go b/cmd/deploy.go index 3868910..f102ebd 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -107,8 +107,8 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R "--clusterrole=cluster-admin", "--serviceaccount=agent-sandbox-system:agent-sandbox-controller") - // RBAC for launcher (from deploy/rbac.yaml) - if err := kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "deploy", "rbac.yaml")); err != nil { + // RBAC for launcher + if err := kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "gateways", "ocp", "addons", "rbac.yaml")); err != nil { return fmt.Errorf("applying launcher RBAC: %w", err) } @@ -125,7 +125,7 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R helmArgs := []string{ "upgrade", "--install", "openshell", chartOCI, "--version", chartVersion, - "--values", filepath.Join(harnessDir, "values-ocp.yaml"), + "--values", filepath.Join(harnessDir, "gateways", "ocp", "helm", "values.yaml"), "--set", "server.sandboxImage=" + sandboxImage, "--set", "pkiInitJob.serverDnsNames[0]=" + routeHost, } @@ -145,10 +145,10 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R return fmt.Errorf("gateway rollout failed: %w", err) } - // Step 5: Route (from deploy/route.yaml) + // Step 5: Route status.Step(5, "Creating OpenShift route") if err := kc.RunKubectlQuiet(ctx, "get", "route", "gateway"); err != nil { - kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "deploy", "route.yaml")) + kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "gateways", "ocp", "addons", "route.yaml")) } fmt.Printf(" Route: %s\n", routeHost) diff --git a/cmd/new.go b/cmd/new.go index 9cfec17..81fe8c4 100644 --- a/cmd/new.go +++ b/cmd/new.go @@ -174,7 +174,7 @@ func newRemote(harnessDir string, gw gateway.Gateway, profileName, sandboxName s "restartPolicy": "Never", "containers": []map[string]any{{ "name": "launcher", - "image": "ghcr.io/robbycochran/harness-openshell:launcher", + "image": envOr("LAUNCHER_IMAGE", "ghcr.io/robbycochran/harness-openshell:launcher"), "imagePullPolicy": "Always", "env": []map[string]any{ {"name": "GATEWAY_ENDPOINT", "value": "https://openshell.openshell.svc.cluster.local:8080"}, @@ -332,10 +332,18 @@ func newLocal(opts newLocalOpts) error { // 6. Build command var sandboxCmd []string - if opts.noTTY { - sandboxCmd = []string{"bash", "/sandbox/startup.sh"} + if cfg.Startup != "" { + if opts.noTTY { + sandboxCmd = []string{"bash", "-c", fmt.Sprintf(". %s", cfg.Startup)} + } else { + sandboxCmd = []string{"bash", "-c", fmt.Sprintf(". %s && exec %s", cfg.Startup, cfg.Command)} + } } else { - sandboxCmd = []string{"bash", "-c", fmt.Sprintf(". /sandbox/startup.sh && exec %s", cfg.Command)} + if opts.noTTY { + sandboxCmd = []string{"true"} + } else { + sandboxCmd = []string{"bash", "-c", fmt.Sprintf("exec %s", cfg.Command)} + } } // 7. Create sandbox with retry diff --git a/docs/design.md b/docs/design.md index 53ca0b0..1362e53 100644 --- a/docs/design.md +++ b/docs/design.md @@ -287,6 +287,193 @@ point proto types ARE the request payloads and the migration pays for itself. TOML into proto-generated structs internally with a ~50-line mapping layer. Users keep writing TOML. No textproto, no format change. +## Gateway configs + +Gateway configs describe how to deploy and connect to a gateway. Each +config is a directory under `gateways/` with a consistent structure: + +``` +gateways/ + local/ + gateway.toml # type = "local", no deploy needed + kind/ + gateway.toml # type = "remote", platform = "k8s", NodePort + helm/values.yaml # Helm overrides for kind + ocp/ + gateway.toml # type = "remote", platform = "ocp", Route + helm/values.yaml # Helm overrides for OpenShift + addons/ + rbac.yaml # launcher ServiceAccount + Role + route.yaml # OpenShift Route for external access +``` + +### `gateway.toml` schema + +```toml +[gateway] +type = "remote" # "local" or "remote" +platform = "ocp" # "ocp", "k8s", or empty (auto-detect) +service = "route" # "route", "nodeport", "loadbalancer" + +[helm] +values = "values.yaml" # relative to helm/ subdir + +[addons] +manifests = [ # applied after Helm install + "addons/rbac.yaml", + "addons/route.yaml", +] +``` + +### Deploy behavior by config + +`harness deploy [GATEWAY_NAME]` reads the gateway config and acts on it: + +| Field | Effect | +|-------|--------| +| `type = "local"` | Verify local gateway is running, skip Helm | +| `platform = "ocp"` | Run SCC grants via `oc`, query appsDomain | +| `platform = "k8s"` | Skip SCCs, use standard k8s networking | +| `service = "route"` | Create OpenShift Route, derive endpoint from appsDomain | +| `service = "nodeport"` | Extract NodePort, configure endpoint as `host:port` | +| `service = "loadbalancer"` | Wait for external IP, configure endpoint | +| `helm.values` | Pass `--values` to Helm install | +| `addons.manifests` | `kubectl apply -f` each manifest after Helm | + +### Sandbox profiles vs gateway configs + +Two orthogonal axes — *what* to run vs *where* to run it: + +``` +gateway config (where) × sandbox profile (what) +────────────────────── ────────────────────── +local default (full tooling) +kind ci (minimal, no providers) +ocp research (different model) +k8s-gke ... +``` + +Any sandbox profile works on any gateway. The harness resolves the +active gateway and creates the sandbox through it. + +### Custom gateways + +To add a new gateway target: copy an existing directory, edit +`gateway.toml`, add Helm values and addon manifests as needed. +No code changes required. + +## Container runtime + +The harness doesn't specify or manage the container runtime (podman vs +docker). The openshell gateway auto-detects the driver at startup +(Kubernetes → Podman → Docker). The driver can be overridden via: + +``` +OPENSHELL_DRIVERS=podman openshell-gateway +OPENSHELL_DRIVERS=docker openshell-gateway +``` + +Or in a gateway config file (`--config` / `OPENSHELL_GATEWAY_CONFIG`). + +The harness refers to "local" (vs "remote/OCP") rather than "podman" +since the runtime is an openshell concern, not a harness concern. + +The driver is not exposed via the gRPC API (by design — clients are +abstracted from the compute layer). However, the gateway logs the +driver at startup: + +``` +INFO openshell_server: Using compute driver driver=podman +INFO openshell_driver_podman::driver: Connected to Podman cgroup_version=v2 ... +``` + +Log location (macOS/Homebrew): +`/opt/homebrew/var/log/openshell/openshell-gateway.out.log` + +`harness status` can grep the gateway log for "Using compute driver" +to report the active driver. This is best-effort — if the log is +unavailable, status just omits the driver line. + +## OCP vs vanilla Kubernetes + +The upstream openshell Helm chart is vanilla-k8s-first. OpenShift is the +variant that requires extra steps. The harness handles both via gateway +configs. + +### What OCP needs beyond vanilla k8s + +| Step | OCP | Vanilla k8s | Why | +|------|-----|-------------|-----| +| **SCCs** | Grant `privileged` SCC to openshell SAs via `oc adm policy` | Not needed — no SCC concept | OCP blocks pods from running as certain UIDs by default | +| **Helm values** | Null out `runAsUser` and `runAsNonRoot` (SCC assigns UID) | Chart defaults work (`runAsUser: 1000`) | OCP SCC admission rejects the chart's hardcoded security context | +| **Networking** | OpenShift Route (TLS passthrough) | Ingress, LoadBalancer, or NodePort | Route is OCP-only; vanilla k8s uses standard networking | +| **Apps domain** | Query `ingresses.config.openshift.io/cluster` for wildcard domain | User-provided or derived from service | OCP-only API | +| **PKI init job** | Works if privileged SCC is granted | Works by default | OCP's SCC would block it without the grant | + +### What's the same on both + +- Namespace creation with Pod Security Standards labels +- Sandbox CRD installation (kubernetes-sigs/agent-sandbox) +- Helm chart install (same chart, different values) +- RBAC for launcher (standard k8s ServiceAccount + Role) +- mTLS certificate generation and extraction +- Launcher Job spec +- Provider registration and sandbox creation + +## Security: TLS and authentication + +### Three network paths + +``` +laptop ──[mTLS over Route/Ingress]──▶ gateway (in cluster) +launcher Job ──[mTLS over cluster DNS]──▶ gateway (in cluster) +gateway ──[container runtime]──▶ sandbox pod +``` + +All external traffic is mTLS-encrypted. The launcher Job mounts the +`openshell-client-tls` Secret for in-cluster mTLS to the gateway. + +### Auth roadmap + +| Stage | Auth mechanism | Use case | +|-------|---------------|----------| +| **Now** | mTLS certificates extracted from cluster | Single user, CI | +| **Next** | oauth-proxy sidecar + OpenShift OAuth | Team — OCP users log in with cluster creds | +| **Future** | OIDC (Keycloak, Dex, external IdP) | Multi-cluster, external users | + +### Current approach: mTLS-as-auth + +The Helm chart's PKI init job generates server and client certificates. +The harness extracts the client cert from `openshell-client-tls` Secret +and configures the CLI with it. mTLS serves as both transport encryption +and user authentication. + +This is secure (encrypted + authenticated) but single-user — whoever has +the client cert has full access. + +### Next: oauth-proxy sidecar on OCP + +OpenShift has a built-in OAuth server. The cluster already has it running +(`openshift-authentication` namespace). Any user who can `oc login` can +authenticate. + +The approach: +1. Deploy an oauth-proxy sidecar alongside the gateway (addon manifest) +2. Route points at the proxy port, proxy forwards to gateway +3. Gateway runs with `allowUnauthenticatedUsers: true` (trusts the proxy) +4. CLI registers with bare `https://` (browser-based login flow) +5. No external IdP needed — uses the cluster's own user directory + +This requires no internal approval for new services — oauth-proxy is a +standard OCP pattern and the OAuth server is already running. + +### Future: OIDC + +For production multi-cluster deployments, configure the gateway with +`--oidc-issuer` pointing at Keycloak, Dex, Okta, or any OIDC provider. +The CLI uses `--oidc-issuer` and `--oidc-client-id` on `gateway add` +and does a browser-based OIDC login flow. + ## Open questions - Should `harness init` (from release-plan.md) be a separate command or diff --git a/gateways/kind/gateway.toml b/gateways/kind/gateway.toml new file mode 100644 index 0000000..2237ba6 --- /dev/null +++ b/gateways/kind/gateway.toml @@ -0,0 +1,16 @@ +# kind gateway — deploys openshell to a local kind cluster. +# +# Prerequisites: +# - kind installed +# - Cluster created: kind create cluster --name openshell +# +# Uses NodePort for access (no Ingress controller needed). +# No user authentication — suitable for CI and local dev only. + +[gateway] +type = "remote" +platform = "k8s" +service = "nodeport" + +[helm] +# Chart defaults work for vanilla k8s — no values override needed. diff --git a/gateways/kind/helm/values.yaml b/gateways/kind/helm/values.yaml new file mode 100644 index 0000000..d01ce73 --- /dev/null +++ b/gateways/kind/helm/values.yaml @@ -0,0 +1,11 @@ +# kind gateway Helm values. +# +# Chart defaults are vanilla-k8s-friendly — this file only overrides +# what's needed for kind specifically. + +service: + type: NodePort + +server: + auth: + allowUnauthenticatedUsers: true diff --git a/gateways/local/gateway.toml b/gateways/local/gateway.toml new file mode 100644 index 0000000..3999b3f --- /dev/null +++ b/gateways/local/gateway.toml @@ -0,0 +1,10 @@ +# Local gateway — runs on your machine via podman or docker. +# +# The openshell gateway must be installed and running: +# macOS: brew install openshell && brew services start openshell +# Linux: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh +# +# The harness verifies the gateway is reachable but does not manage its lifecycle. + +[gateway] +type = "local" diff --git a/deploy/rbac.yaml b/gateways/ocp/addons/rbac.yaml similarity index 100% rename from deploy/rbac.yaml rename to gateways/ocp/addons/rbac.yaml diff --git a/deploy/route.yaml b/gateways/ocp/addons/route.yaml similarity index 100% rename from deploy/route.yaml rename to gateways/ocp/addons/route.yaml diff --git a/gateways/ocp/gateway.toml b/gateways/ocp/gateway.toml new file mode 100644 index 0000000..fc6d59b --- /dev/null +++ b/gateways/ocp/gateway.toml @@ -0,0 +1,21 @@ +# OpenShift gateway — deploys openshell to an OCP cluster. +# +# Prerequisites: +# - oc login to the target cluster +# - KUBECONFIG set or default context pointing to OCP +# +# Addons applied: +# - addons/rbac.yaml — launcher ServiceAccount + Role +# - addons/route.yaml — OpenShift Route for external access +# - SCC grants for openshell service accounts (via oc CLI) + +[gateway] +type = "remote" +platform = "ocp" +service = "route" + +[helm] +values = "values.yaml" + +[addons] +manifests = ["addons/rbac.yaml", "addons/route.yaml"] diff --git a/values-ocp.yaml b/gateways/ocp/helm/values.yaml similarity index 100% rename from values-ocp.yaml rename to gateways/ocp/helm/values.yaml diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 9b6c3c4..1e15e41 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -21,6 +21,7 @@ type Config struct { Name string `toml:"name"` From string `toml:"from"` Command string `toml:"command"` + Startup string `toml:"startup"` Keep *bool `toml:"keep"` Providers []string `toml:"providers"` Env map[string]string `toml:"env"` diff --git a/profiles/ci.toml b/profiles/ci.toml new file mode 100644 index 0000000..fcd328d --- /dev/null +++ b/profiles/ci.toml @@ -0,0 +1,10 @@ +# CI integration test profile. +# +# Minimal sandbox with no providers and no credentials. +# Uses the public community base image (no registry auth needed). + +name = "ci-test" +from = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +command = "bash" +keep = true +providers = [] diff --git a/profiles/default.toml b/profiles/default.toml index 8588892..91d4f64 100644 --- a/profiles/default.toml +++ b/profiles/default.toml @@ -7,6 +7,7 @@ name = "agent" from = "ghcr.io/robbycochran/harness-openshell:sandbox" command = "claude --bare" +startup = "/sandbox/startup.sh" keep = true providers = ["github", "vertex-local", "atlassian"] diff --git a/sandbox/launcher/main.go b/sandbox/launcher/main.go index 17f0abb..3defe2a 100644 --- a/sandbox/launcher/main.go +++ b/sandbox/launcher/main.go @@ -44,13 +44,21 @@ func parseConfig(path string) (*Config, error) { } func configureGateway(endpoint, mtlsDir, cli string) error { - certFile := filepath.Join(mtlsDir, "tls.crt") - if _, err := os.Stat(certFile); err != nil { - fmt.Println(" No mTLS certs, using insecure mode") + requiredCerts := []string{"ca.crt", "tls.crt", "tls.key"} + var missing []string + for _, name := range requiredCerts { + if _, err := os.Stat(filepath.Join(mtlsDir, name)); err != nil { + missing = append(missing, name) + } + } + if len(missing) > 0 { + fmt.Fprintf(os.Stderr, "WARNING: mTLS certs missing from %s: %v\n", mtlsDir, missing) + fmt.Fprintf(os.Stderr, "WARNING: falling back to INSECURE mode — gateway connection is NOT encrypted\n") os.Setenv("OPENSHELL_GATEWAY_ENDPOINT", endpoint) os.Setenv("OPENSHELL_GATEWAY_INSECURE", "true") return nil } + fmt.Println(" ✓ mTLS certs found — using encrypted connection") httpEndpoint := strings.Replace(endpoint, "https:", "http:", 1) diff --git a/test/test-flow.sh b/test/test-flow.sh index 042fc3c..dfe422f 100755 --- a/test/test-flow.sh +++ b/test/test-flow.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# End-to-end validation for podman and OCP flows. +# End-to-end validation for local and OCP flows. # # Usage: -# ./test-flow.sh podman # quick: deploy + providers + teardown -# ./test-flow.sh podman --full # full: + sandbox + verify integrations +# ./test-flow.sh local # quick: deploy + providers + teardown +# ./test-flow.sh local --full # full: + sandbox + verify integrations +# ./test-flow.sh local --full --no-providers --profile=ci # CI mode (no creds) # ./test-flow.sh ocp [--full] # OCP variants # ./test-flow.sh ocp --full --reuse-gateway # skip deploy/teardown-k8s (~50s vs ~130s) # ./test-flow.sh all [--full] # both platforms @@ -23,11 +24,15 @@ fi TARGET="" FULL=false REUSE_GATEWAY=false +NO_PROVIDERS=false +PROFILE="default" for arg in "$@"; do case "$arg" in --full) FULL=true ;; --reuse-gateway) REUSE_GATEWAY=true ;; + --no-providers) NO_PROVIDERS=true ;; + --profile=*) PROFILE="${arg#--profile=}" ;; -*) ;; *) [[ -z "$TARGET" ]] && TARGET="$arg" ;; esac @@ -117,6 +122,14 @@ sandbox_verify() { printf " ✓ %-35s\n" "sandbox ready" ((PASS++)) + # Basic exec works + step "sandbox: exec" "$CLI" sandbox exec --name "$name" -- echo "hello" + + if $NO_PROVIDERS; then + return + fi + + # Provider-dependent checks (require credentials + inference) step "sandbox: env vars" "$CLI" sandbox exec --name "$name" -- bash -c 'test -n "$ANTHROPIC_BASE_URL"' step "sandbox: gws creds" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.config/openshell/credentials.json step "sandbox: mcp config" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.mcp.json @@ -154,31 +167,39 @@ test_errors() { echo "" } -# ── Podman flow ────────────────────────────────────────────────────── +# ── Local flow ─────────────────────────────────────────────────────── -test_podman() { +test_local() { local mode="quick" $FULL && mode="full" - echo "=== test-flow: podman ($mode) ===" + $NO_PROVIDERS && mode="$mode, no-providers" + echo "=== test-flow: local ($mode) ===" step "teardown" "$HARNESS" teardown --sandboxes --providers step "deploy" "$HARNESS" deploy --local - step "setup providers" "$HARNESS" providers - step "gateway reachable" "$CLI" inference get - check_providers + + if ! $NO_PROVIDERS; then + step "setup providers" "$HARNESS" providers + step "gateway reachable" "$CLI" inference get + check_providers + else + step "gateway reachable" "$HARNESS" deploy --local + fi if $FULL; then local sandbox_name="test-agent" - step_output "sandbox create" "$HARNESS" new --local --name "$sandbox_name" --no-tty + step_output "sandbox create" "$HARNESS" new --local --name "$sandbox_name" --profile "$PROFILE" --no-tty sandbox_verify "$sandbox_name" step "sandbox delete" "$CLI" sandbox delete "$sandbox_name" - # Missing providers scenario - echo "" - echo "=== test: missing providers ===" - step "teardown providers" "$HARNESS" teardown --providers - step_output "new with no providers" "$HARNESS" new --local --name test-noprov --no-tty - step "cleanup" "$HARNESS" teardown --sandboxes + if ! $NO_PROVIDERS; then + # Missing providers scenario + echo "" + echo "=== test: missing providers ===" + step "teardown providers" "$HARNESS" teardown --providers + step_output "new with no providers" "$HARNESS" new --local --name test-noprov --no-tty + step "cleanup" "$HARNESS" teardown --sandboxes + fi fi step "teardown (clean)" "$HARNESS" teardown --sandboxes --providers @@ -238,9 +259,9 @@ test_ocp() { test_errors case "$TARGET" in - podman) test_podman ;; + local|podman) test_local ;; ocp) test_ocp ;; - all) test_podman; echo ""; test_ocp ;; + all) test_local; echo ""; test_ocp ;; *) echo "Unknown target: $TARGET" echo "Usage: $0 [--full]"