From de091f6c95d69f92f0c96477eb753647692ac974 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:10:39 -0700 Subject: [PATCH 01/11] X-Smart-Branch-Parent: main From 28b6a2b1713947bad49520e8ccbe58533866dbb3 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:18:58 -0700 Subject: [PATCH 02/11] ci: add podman integration tests on GHA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add profiles/ci.toml — minimal profile with no providers, uses public community base image (no registry auth needed) - Add startup field to profile Config — controls whether startup.sh runs (default profile sets it, CI profile omits it) - Add --no-providers and --profile flags to test-flow.sh - sandbox_verify skips provider-dependent checks with --no-providers - Add .github/workflows/integration.yml — runs podman integration tests on every PR and push to main Tested locally: 11/11 CI profile, 20/20 default profile. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/integration.yml | 35 +++++++++++++++++++++++++++ cmd/new.go | 14 ++++++++--- internal/profile/profile.go | 1 + profiles/ci.toml | 10 ++++++++ profiles/default.toml | 1 + test/test-flow.sh | 40 +++++++++++++++++++++++-------- 6 files changed, 88 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/integration.yml create mode 100644 profiles/ci.toml diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..9192f0a --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,35 @@ +name: Integration + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + podman: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Build harness CLI + run: make cli + + - name: Install openshell + run: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh + + - name: Wait for gateway + run: | + for i in $(seq 1 30); do + openshell inference get &>/dev/null && break + sleep 1 + done + openshell gateway list + + - name: Run integration tests + run: ./test/test-flow.sh podman --full --no-providers --profile=ci diff --git a/cmd/new.go b/cmd/new.go index 9cfec17..491b792 100644 --- a/cmd/new.go +++ b/cmd/new.go @@ -332,10 +332,18 @@ func newLocal(opts newLocalOpts) error { // 6. Build command var sandboxCmd []string - if opts.noTTY { - sandboxCmd = []string{"bash", "/sandbox/startup.sh"} + if cfg.Startup != "" { + if opts.noTTY { + sandboxCmd = []string{"bash", "-c", fmt.Sprintf(". %s", cfg.Startup)} + } else { + sandboxCmd = []string{"bash", "-c", fmt.Sprintf(". %s && exec %s", cfg.Startup, cfg.Command)} + } } else { - sandboxCmd = []string{"bash", "-c", fmt.Sprintf(". /sandbox/startup.sh && exec %s", cfg.Command)} + if opts.noTTY { + sandboxCmd = []string{"true"} + } else { + sandboxCmd = []string{"bash", "-c", fmt.Sprintf("exec %s", cfg.Command)} + } } // 7. Create sandbox with retry diff --git a/internal/profile/profile.go b/internal/profile/profile.go index 9b6c3c4..1e15e41 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -21,6 +21,7 @@ type Config struct { Name string `toml:"name"` From string `toml:"from"` Command string `toml:"command"` + Startup string `toml:"startup"` Keep *bool `toml:"keep"` Providers []string `toml:"providers"` Env map[string]string `toml:"env"` diff --git a/profiles/ci.toml b/profiles/ci.toml new file mode 100644 index 0000000..fcd328d --- /dev/null +++ b/profiles/ci.toml @@ -0,0 +1,10 @@ +# CI integration test profile. +# +# Minimal sandbox with no providers and no credentials. +# Uses the public community base image (no registry auth needed). + +name = "ci-test" +from = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest" +command = "bash" +keep = true +providers = [] diff --git a/profiles/default.toml b/profiles/default.toml index 8588892..91d4f64 100644 --- a/profiles/default.toml +++ b/profiles/default.toml @@ -7,6 +7,7 @@ name = "agent" from = "ghcr.io/robbycochran/harness-openshell:sandbox" command = "claude --bare" +startup = "/sandbox/startup.sh" keep = true providers = ["github", "vertex-local", "atlassian"] diff --git a/test/test-flow.sh b/test/test-flow.sh index 042fc3c..d54e0ef 100755 --- a/test/test-flow.sh +++ b/test/test-flow.sh @@ -23,11 +23,15 @@ fi TARGET="" FULL=false REUSE_GATEWAY=false +NO_PROVIDERS=false +PROFILE="default" for arg in "$@"; do case "$arg" in --full) FULL=true ;; --reuse-gateway) REUSE_GATEWAY=true ;; + --no-providers) NO_PROVIDERS=true ;; + --profile=*) PROFILE="${arg#--profile=}" ;; -*) ;; *) [[ -z "$TARGET" ]] && TARGET="$arg" ;; esac @@ -117,6 +121,14 @@ sandbox_verify() { printf " ✓ %-35s\n" "sandbox ready" ((PASS++)) + # Basic exec works + step "sandbox: exec" "$CLI" sandbox exec --name "$name" -- echo "hello" + + if $NO_PROVIDERS; then + return + fi + + # Provider-dependent checks (require credentials + inference) step "sandbox: env vars" "$CLI" sandbox exec --name "$name" -- bash -c 'test -n "$ANTHROPIC_BASE_URL"' step "sandbox: gws creds" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.config/openshell/credentials.json step "sandbox: mcp config" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.mcp.json @@ -159,26 +171,34 @@ test_errors() { test_podman() { local mode="quick" $FULL && mode="full" + $NO_PROVIDERS && mode="$mode, no-providers" echo "=== test-flow: podman ($mode) ===" step "teardown" "$HARNESS" teardown --sandboxes --providers step "deploy" "$HARNESS" deploy --local - step "setup providers" "$HARNESS" providers - step "gateway reachable" "$CLI" inference get - check_providers + + if ! $NO_PROVIDERS; then + step "setup providers" "$HARNESS" providers + step "gateway reachable" "$CLI" inference get + check_providers + else + step "gateway reachable" "$HARNESS" deploy --local + fi if $FULL; then local sandbox_name="test-agent" - step_output "sandbox create" "$HARNESS" new --local --name "$sandbox_name" --no-tty + step_output "sandbox create" "$HARNESS" new --local --name "$sandbox_name" --profile "$PROFILE" --no-tty sandbox_verify "$sandbox_name" step "sandbox delete" "$CLI" sandbox delete "$sandbox_name" - # Missing providers scenario - echo "" - echo "=== test: missing providers ===" - step "teardown providers" "$HARNESS" teardown --providers - step_output "new with no providers" "$HARNESS" new --local --name test-noprov --no-tty - step "cleanup" "$HARNESS" teardown --sandboxes + if ! $NO_PROVIDERS; then + # Missing providers scenario + echo "" + echo "=== test: missing providers ===" + step "teardown providers" "$HARNESS" teardown --providers + step_output "new with no providers" "$HARNESS" new --local --name test-noprov --no-tty + step "cleanup" "$HARNESS" teardown --sandboxes + fi fi step "teardown (clean)" "$HARNESS" teardown --sandboxes --providers From a7d76433fef20398f58a4a3b13b852fa8f747ea4 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:25:47 -0700 Subject: [PATCH 03/11] =?UTF-8?q?refactor:=20rename=20podman=20=E2=86=92?= =?UTF-8?q?=20local=20throughout,=20document=20driver=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness doesn't manage the container runtime — openshell auto-detects the driver (podman/docker). Rename test targets, workflow jobs, and Makefile from 'podman' to 'local' since the runtime is an openshell concern. Keep 'podman' as a test-flow.sh alias for backward compat. Add container runtime section to design.md documenting OPENSHELL_DRIVERS env var and gateway --config for driver override. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/integration.yml | 4 ++-- Makefile | 14 +++++++------- docs/design.md | 18 ++++++++++++++++++ test/test-flow.sh | 17 +++++++++-------- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9192f0a..622fa5c 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,7 +9,7 @@ permissions: contents: read jobs: - podman: + local: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -32,4 +32,4 @@ jobs: openshell gateway list - name: Run integration tests - run: ./test/test-flow.sh podman --full --no-providers --profile=ci + run: ./test/test-flow.sh local --full --no-providers --profile=ci diff --git a/Makefile b/Makefile index 7c5aa83..d6319ad 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SANDBOX_IMAGE := $(REGISTRY):sandbox LAUNCHER_IMAGE := $(REGISTRY):launcher .PHONY: cli sandbox push-sandbox cli-launcher launcher push-launcher \ - vet lint test-unit test test-podman test-ocp test-all validate clean help + vet lint test-unit test test-local test-ocp test-all validate clean help ## ── CLI ────────────────────────────────────────────────────────────── @@ -75,9 +75,9 @@ test-unit: test: cli sandbox push-launcher ./test/test-flow.sh all --full -## Podman only (full lifecycle) -test-podman: cli sandbox push-launcher - ./test/test-flow.sh podman --full +## Local gateway (full lifecycle) +test-local: cli + ./test/test-flow.sh local --full ## OCP only (full lifecycle) test-ocp: cli sandbox push-launcher @@ -87,7 +87,7 @@ test-ocp: cli sandbox push-launcher test-all: cli sandbox push-launcher ./test/test-flow.sh all --full -## Full validation: unit tests + bats + integration (podman + OCP) +## Full validation: unit tests + bats + integration (local + OCP) ## Run this before every commit. validate: cli sandbox push-launcher @echo "=== Unit tests ===" @@ -97,8 +97,8 @@ validate: cli sandbox push-launcher @echo "=== Bats ===" bats test/preflight.bats @echo "" - @echo "=== Integration: podman ===" - ./test/test-flow.sh podman --full + @echo "=== Integration: local ===" + ./test/test-flow.sh local --full @echo "" @echo "=== Integration: OCP ===" ./test/test-flow.sh ocp --full diff --git a/docs/design.md b/docs/design.md index 53ca0b0..f84b880 100644 --- a/docs/design.md +++ b/docs/design.md @@ -287,6 +287,24 @@ point proto types ARE the request payloads and the migration pays for itself. TOML into proto-generated structs internally with a ~50-line mapping layer. Users keep writing TOML. No textproto, no format change. +## Container runtime + +The harness doesn't specify or manage the container runtime (podman vs +docker). The openshell gateway auto-detects the driver at startup +(Kubernetes → Podman → Docker). The driver can be overridden via: + +``` +OPENSHELL_DRIVERS=podman openshell-gateway +OPENSHELL_DRIVERS=docker openshell-gateway +``` + +Or in a gateway config file (`--config` / `OPENSHELL_GATEWAY_CONFIG`). + +The harness refers to "local" (vs "remote/OCP") rather than "podman" +since the runtime is an openshell concern, not a harness concern. +`harness status` should report the detected driver when displaying +gateway info (query from `openshell status`). + ## Open questions - Should `harness init` (from release-plan.md) be a separate command or diff --git a/test/test-flow.sh b/test/test-flow.sh index d54e0ef..dfe422f 100755 --- a/test/test-flow.sh +++ b/test/test-flow.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# End-to-end validation for podman and OCP flows. +# End-to-end validation for local and OCP flows. # # Usage: -# ./test-flow.sh podman # quick: deploy + providers + teardown -# ./test-flow.sh podman --full # full: + sandbox + verify integrations +# ./test-flow.sh local # quick: deploy + providers + teardown +# ./test-flow.sh local --full # full: + sandbox + verify integrations +# ./test-flow.sh local --full --no-providers --profile=ci # CI mode (no creds) # ./test-flow.sh ocp [--full] # OCP variants # ./test-flow.sh ocp --full --reuse-gateway # skip deploy/teardown-k8s (~50s vs ~130s) # ./test-flow.sh all [--full] # both platforms @@ -166,13 +167,13 @@ test_errors() { echo "" } -# ── Podman flow ────────────────────────────────────────────────────── +# ── Local flow ─────────────────────────────────────────────────────── -test_podman() { +test_local() { local mode="quick" $FULL && mode="full" $NO_PROVIDERS && mode="$mode, no-providers" - echo "=== test-flow: podman ($mode) ===" + echo "=== test-flow: local ($mode) ===" step "teardown" "$HARNESS" teardown --sandboxes --providers step "deploy" "$HARNESS" deploy --local @@ -258,9 +259,9 @@ test_ocp() { test_errors case "$TARGET" in - podman) test_podman ;; + local|podman) test_local ;; ocp) test_ocp ;; - all) test_podman; echo ""; test_ocp ;; + all) test_local; echo ""; test_ocp ;; *) echo "Unknown target: $TARGET" echo "Usage: $0 [--full]" From 3e50efb91a41c59b7ba87515fb2bbf4f895d5e5e Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:29:17 -0700 Subject: [PATCH 04/11] docs: document gateway driver detection from logs The openshell gateway logs 'Using compute driver driver=podman' at startup but doesn't expose it via gRPC API (by design). Document the log location and how harness status can extract it. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/design.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/design.md b/docs/design.md index f84b880..16dfb46 100644 --- a/docs/design.md +++ b/docs/design.md @@ -302,8 +302,22 @@ Or in a gateway config file (`--config` / `OPENSHELL_GATEWAY_CONFIG`). The harness refers to "local" (vs "remote/OCP") rather than "podman" since the runtime is an openshell concern, not a harness concern. -`harness status` should report the detected driver when displaying -gateway info (query from `openshell status`). + +The driver is not exposed via the gRPC API (by design — clients are +abstracted from the compute layer). However, the gateway logs the +driver at startup: + +``` +INFO openshell_server: Using compute driver driver=podman +INFO openshell_driver_podman::driver: Connected to Podman cgroup_version=v2 ... +``` + +Log location (macOS/Homebrew): +`/opt/homebrew/var/log/openshell/openshell-gateway.out.log` + +`harness status` can grep the gateway log for "Using compute driver" +to report the active driver. This is best-effort — if the log is +unavailable, status just omits the driver line. ## Open questions From c2ec07266a55c1a110cd7d4bbd1dfd057e0f16a0 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:47:12 -0700 Subject: [PATCH 05/11] refactor: introduce gateways/ directory for gateway deployment configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move OCP deploy artifacts from deploy/ and values-ocp.yaml into gateways/ocp/ with addons subdirectory. Add gateway config stubs for local, ocp, and kind targets. Structure: gateways/local.toml — local podman/docker gateway gateways/ocp/gateway.toml — OpenShift deployment config gateways/ocp/values.yaml — Helm values (was values-ocp.yaml) gateways/ocp/addons/rbac.yaml — launcher RBAC (was deploy/rbac.yaml) gateways/ocp/addons/route.yaml — OCP Route (was deploy/route.yaml) gateways/kind/gateway.toml — kind cluster config (NodePort, no auth) Code references updated in cmd/deploy.go. Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/deploy.go | 10 +++++----- gateways/kind/gateway.toml | 16 ++++++++++++++++ gateways/local.toml | 10 ++++++++++ {deploy => gateways/ocp/addons}/rbac.yaml | 0 {deploy => gateways/ocp/addons}/route.yaml | 0 gateways/ocp/gateway.toml | 21 +++++++++++++++++++++ values-ocp.yaml => gateways/ocp/values.yaml | 0 7 files changed, 52 insertions(+), 5 deletions(-) create mode 100644 gateways/kind/gateway.toml create mode 100644 gateways/local.toml rename {deploy => gateways/ocp/addons}/rbac.yaml (100%) rename {deploy => gateways/ocp/addons}/route.yaml (100%) create mode 100644 gateways/ocp/gateway.toml rename values-ocp.yaml => gateways/ocp/values.yaml (100%) diff --git a/cmd/deploy.go b/cmd/deploy.go index 3868910..b76bed7 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -107,8 +107,8 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R "--clusterrole=cluster-admin", "--serviceaccount=agent-sandbox-system:agent-sandbox-controller") - // RBAC for launcher (from deploy/rbac.yaml) - if err := kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "deploy", "rbac.yaml")); err != nil { + // RBAC for launcher + if err := kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "gateways", "ocp", "addons", "rbac.yaml")); err != nil { return fmt.Errorf("applying launcher RBAC: %w", err) } @@ -125,7 +125,7 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R helmArgs := []string{ "upgrade", "--install", "openshell", chartOCI, "--version", chartVersion, - "--values", filepath.Join(harnessDir, "values-ocp.yaml"), + "--values", filepath.Join(harnessDir, "gateways", "ocp", "values.yaml"), "--set", "server.sandboxImage=" + sandboxImage, "--set", "pkiInitJob.serverDnsNames[0]=" + routeHost, } @@ -145,10 +145,10 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R return fmt.Errorf("gateway rollout failed: %w", err) } - // Step 5: Route (from deploy/route.yaml) + // Step 5: Route status.Step(5, "Creating OpenShift route") if err := kc.RunKubectlQuiet(ctx, "get", "route", "gateway"); err != nil { - kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "deploy", "route.yaml")) + kc.RunKubectlPassthrough(ctx, "apply", "-f", filepath.Join(harnessDir, "gateways", "ocp", "addons", "route.yaml")) } fmt.Printf(" Route: %s\n", routeHost) diff --git a/gateways/kind/gateway.toml b/gateways/kind/gateway.toml new file mode 100644 index 0000000..2237ba6 --- /dev/null +++ b/gateways/kind/gateway.toml @@ -0,0 +1,16 @@ +# kind gateway — deploys openshell to a local kind cluster. +# +# Prerequisites: +# - kind installed +# - Cluster created: kind create cluster --name openshell +# +# Uses NodePort for access (no Ingress controller needed). +# No user authentication — suitable for CI and local dev only. + +[gateway] +type = "remote" +platform = "k8s" +service = "nodeport" + +[helm] +# Chart defaults work for vanilla k8s — no values override needed. diff --git a/gateways/local.toml b/gateways/local.toml new file mode 100644 index 0000000..3999b3f --- /dev/null +++ b/gateways/local.toml @@ -0,0 +1,10 @@ +# Local gateway — runs on your machine via podman or docker. +# +# The openshell gateway must be installed and running: +# macOS: brew install openshell && brew services start openshell +# Linux: curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh +# +# The harness verifies the gateway is reachable but does not manage its lifecycle. + +[gateway] +type = "local" diff --git a/deploy/rbac.yaml b/gateways/ocp/addons/rbac.yaml similarity index 100% rename from deploy/rbac.yaml rename to gateways/ocp/addons/rbac.yaml diff --git a/deploy/route.yaml b/gateways/ocp/addons/route.yaml similarity index 100% rename from deploy/route.yaml rename to gateways/ocp/addons/route.yaml diff --git a/gateways/ocp/gateway.toml b/gateways/ocp/gateway.toml new file mode 100644 index 0000000..fc6d59b --- /dev/null +++ b/gateways/ocp/gateway.toml @@ -0,0 +1,21 @@ +# OpenShift gateway — deploys openshell to an OCP cluster. +# +# Prerequisites: +# - oc login to the target cluster +# - KUBECONFIG set or default context pointing to OCP +# +# Addons applied: +# - addons/rbac.yaml — launcher ServiceAccount + Role +# - addons/route.yaml — OpenShift Route for external access +# - SCC grants for openshell service accounts (via oc CLI) + +[gateway] +type = "remote" +platform = "ocp" +service = "route" + +[helm] +values = "values.yaml" + +[addons] +manifests = ["addons/rbac.yaml", "addons/route.yaml"] diff --git a/values-ocp.yaml b/gateways/ocp/values.yaml similarity index 100% rename from values-ocp.yaml rename to gateways/ocp/values.yaml From 622f4fc2c2570bef3761910ca65642b6dfb228f1 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:47:45 -0700 Subject: [PATCH 06/11] fix: move local.toml into gateways/local/gateway.toml for consistency Co-Authored-By: Claude Opus 4.6 (1M context) --- gateways/{local.toml => local/gateway.toml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gateways/{local.toml => local/gateway.toml} (100%) diff --git a/gateways/local.toml b/gateways/local/gateway.toml similarity index 100% rename from gateways/local.toml rename to gateways/local/gateway.toml From c0b3bf10597d01c7b961667938e1daeafa3faf22 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:48:50 -0700 Subject: [PATCH 07/11] refactor: add helm/ subdirectory to gateway configs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each remote gateway now has a consistent structure: gateway.toml — gateway config (type, platform, service) helm/values.yaml — Helm values for this gateway addons/ — post-install manifests (OCP only for now) kind gateway gets minimal values (NodePort, no auth). Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/deploy.go | 2 +- gateways/kind/helm/values.yaml | 11 +++++++++++ gateways/ocp/{ => helm}/values.yaml | 0 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 gateways/kind/helm/values.yaml rename gateways/ocp/{ => helm}/values.yaml (100%) diff --git a/cmd/deploy.go b/cmd/deploy.go index b76bed7..f102ebd 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -125,7 +125,7 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kc, clusterRunner k8s.R helmArgs := []string{ "upgrade", "--install", "openshell", chartOCI, "--version", chartVersion, - "--values", filepath.Join(harnessDir, "gateways", "ocp", "values.yaml"), + "--values", filepath.Join(harnessDir, "gateways", "ocp", "helm", "values.yaml"), "--set", "server.sandboxImage=" + sandboxImage, "--set", "pkiInitJob.serverDnsNames[0]=" + routeHost, } diff --git a/gateways/kind/helm/values.yaml b/gateways/kind/helm/values.yaml new file mode 100644 index 0000000..d01ce73 --- /dev/null +++ b/gateways/kind/helm/values.yaml @@ -0,0 +1,11 @@ +# kind gateway Helm values. +# +# Chart defaults are vanilla-k8s-friendly — this file only overrides +# what's needed for kind specifically. + +service: + type: NodePort + +server: + auth: + allowUnauthenticatedUsers: true diff --git a/gateways/ocp/values.yaml b/gateways/ocp/helm/values.yaml similarity index 100% rename from gateways/ocp/values.yaml rename to gateways/ocp/helm/values.yaml From ce5bbfb9b97288ee4ecf006cd73b59a99d845254 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 12:54:20 -0700 Subject: [PATCH 08/11] docs: gateway config system in design.md + CI log export MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the gateway config schema, deploy behavior by config field, sandbox-vs-gateway orthogonality, and custom gateway instructions. Add gateway log export step to local integration CI workflow — captures journalctl logs, openshell status, and gateway list as artifacts for debugging failures. Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/integration.yml | 14 ++++++ docs/design.md | 75 +++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 622fa5c..d0c1d7b 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -33,3 +33,17 @@ jobs: - name: Run integration tests run: ./test/test-flow.sh local --full --no-providers --profile=ci + + - name: Export gateway logs + if: always() + run: | + mkdir -p /tmp/gateway-logs + journalctl --user -u openshell-gateway --no-pager > /tmp/gateway-logs/gateway.log 2>/dev/null || true + openshell status > /tmp/gateway-logs/status.txt 2>&1 || true + openshell gateway list > /tmp/gateway-logs/gateways.txt 2>&1 || true + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: gateway-logs-local + path: /tmp/gateway-logs/ diff --git a/docs/design.md b/docs/design.md index 16dfb46..42cd56f 100644 --- a/docs/design.md +++ b/docs/design.md @@ -287,6 +287,81 @@ point proto types ARE the request payloads and the migration pays for itself. TOML into proto-generated structs internally with a ~50-line mapping layer. Users keep writing TOML. No textproto, no format change. +## Gateway configs + +Gateway configs describe how to deploy and connect to a gateway. Each +config is a directory under `gateways/` with a consistent structure: + +``` +gateways/ + local/ + gateway.toml # type = "local", no deploy needed + kind/ + gateway.toml # type = "remote", platform = "k8s", NodePort + helm/values.yaml # Helm overrides for kind + ocp/ + gateway.toml # type = "remote", platform = "ocp", Route + helm/values.yaml # Helm overrides for OpenShift + addons/ + rbac.yaml # launcher ServiceAccount + Role + route.yaml # OpenShift Route for external access +``` + +### `gateway.toml` schema + +```toml +[gateway] +type = "remote" # "local" or "remote" +platform = "ocp" # "ocp", "k8s", or empty (auto-detect) +service = "route" # "route", "nodeport", "loadbalancer" + +[helm] +values = "values.yaml" # relative to helm/ subdir + +[addons] +manifests = [ # applied after Helm install + "addons/rbac.yaml", + "addons/route.yaml", +] +``` + +### Deploy behavior by config + +`harness deploy [GATEWAY_NAME]` reads the gateway config and acts on it: + +| Field | Effect | +|-------|--------| +| `type = "local"` | Verify local gateway is running, skip Helm | +| `platform = "ocp"` | Run SCC grants via `oc`, query appsDomain | +| `platform = "k8s"` | Skip SCCs, use standard k8s networking | +| `service = "route"` | Create OpenShift Route, derive endpoint from appsDomain | +| `service = "nodeport"` | Extract NodePort, configure endpoint as `host:port` | +| `service = "loadbalancer"` | Wait for external IP, configure endpoint | +| `helm.values` | Pass `--values` to Helm install | +| `addons.manifests` | `kubectl apply -f` each manifest after Helm | + +### Sandbox profiles vs gateway configs + +Two orthogonal axes — *what* to run vs *where* to run it: + +``` +gateway config (where) × sandbox profile (what) +────────────────────── ────────────────────── +local default (full tooling) +kind ci (minimal, no providers) +ocp research (different model) +k8s-gke ... +``` + +Any sandbox profile works on any gateway. The harness resolves the +active gateway and creates the sandbox through it. + +### Custom gateways + +To add a new gateway target: copy an existing directory, edit +`gateway.toml`, add Helm values and addon manifests as needed. +No code changes required. + ## Container runtime The harness doesn't specify or manage the container runtime (podman vs From 9e1244fb8cfd19797cb512ce7f1518855521102f Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 13:07:29 -0700 Subject: [PATCH 09/11] docs: OCP vs vanilla k8s differences, TLS paths, auth roadmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Document the 5 OCP-specific steps vs vanilla k8s (SCCs, Helm values, Route, appsDomain, PKI) - Document three network paths (laptop→gateway, launcher→gateway, gateway→sandbox) and that all use mTLS - Auth roadmap: mTLS-as-auth (now) → oauth-proxy + OCP OAuth (next) → OIDC (future) - Detail oauth-proxy approach: uses existing OCP OAuth server, no external IdP, standard OCP pattern Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/design.md | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/docs/design.md b/docs/design.md index 42cd56f..1362e53 100644 --- a/docs/design.md +++ b/docs/design.md @@ -394,6 +394,86 @@ Log location (macOS/Homebrew): to report the active driver. This is best-effort — if the log is unavailable, status just omits the driver line. +## OCP vs vanilla Kubernetes + +The upstream openshell Helm chart is vanilla-k8s-first. OpenShift is the +variant that requires extra steps. The harness handles both via gateway +configs. + +### What OCP needs beyond vanilla k8s + +| Step | OCP | Vanilla k8s | Why | +|------|-----|-------------|-----| +| **SCCs** | Grant `privileged` SCC to openshell SAs via `oc adm policy` | Not needed — no SCC concept | OCP blocks pods from running as certain UIDs by default | +| **Helm values** | Null out `runAsUser` and `runAsNonRoot` (SCC assigns UID) | Chart defaults work (`runAsUser: 1000`) | OCP SCC admission rejects the chart's hardcoded security context | +| **Networking** | OpenShift Route (TLS passthrough) | Ingress, LoadBalancer, or NodePort | Route is OCP-only; vanilla k8s uses standard networking | +| **Apps domain** | Query `ingresses.config.openshift.io/cluster` for wildcard domain | User-provided or derived from service | OCP-only API | +| **PKI init job** | Works if privileged SCC is granted | Works by default | OCP's SCC would block it without the grant | + +### What's the same on both + +- Namespace creation with Pod Security Standards labels +- Sandbox CRD installation (kubernetes-sigs/agent-sandbox) +- Helm chart install (same chart, different values) +- RBAC for launcher (standard k8s ServiceAccount + Role) +- mTLS certificate generation and extraction +- Launcher Job spec +- Provider registration and sandbox creation + +## Security: TLS and authentication + +### Three network paths + +``` +laptop ──[mTLS over Route/Ingress]──▶ gateway (in cluster) +launcher Job ──[mTLS over cluster DNS]──▶ gateway (in cluster) +gateway ──[container runtime]──▶ sandbox pod +``` + +All external traffic is mTLS-encrypted. The launcher Job mounts the +`openshell-client-tls` Secret for in-cluster mTLS to the gateway. + +### Auth roadmap + +| Stage | Auth mechanism | Use case | +|-------|---------------|----------| +| **Now** | mTLS certificates extracted from cluster | Single user, CI | +| **Next** | oauth-proxy sidecar + OpenShift OAuth | Team — OCP users log in with cluster creds | +| **Future** | OIDC (Keycloak, Dex, external IdP) | Multi-cluster, external users | + +### Current approach: mTLS-as-auth + +The Helm chart's PKI init job generates server and client certificates. +The harness extracts the client cert from `openshell-client-tls` Secret +and configures the CLI with it. mTLS serves as both transport encryption +and user authentication. + +This is secure (encrypted + authenticated) but single-user — whoever has +the client cert has full access. + +### Next: oauth-proxy sidecar on OCP + +OpenShift has a built-in OAuth server. The cluster already has it running +(`openshift-authentication` namespace). Any user who can `oc login` can +authenticate. + +The approach: +1. Deploy an oauth-proxy sidecar alongside the gateway (addon manifest) +2. Route points at the proxy port, proxy forwards to gateway +3. Gateway runs with `allowUnauthenticatedUsers: true` (trusts the proxy) +4. CLI registers with bare `https://` (browser-based login flow) +5. No external IdP needed — uses the cluster's own user directory + +This requires no internal approval for new services — oauth-proxy is a +standard OCP pattern and the OAuth server is already running. + +### Future: OIDC + +For production multi-cluster deployments, configure the gateway with +`--oidc-issuer` pointing at Keycloak, Dex, Okta, or any OIDC provider. +The CLI uses `--oidc-issuer` and `--oidc-client-id` on `gateway add` +and does a browser-based OIDC login flow. + ## Open questions - Should `harness init` (from release-plan.md) be a separate command or From 71271409765bdb5cf0703f93de35f530b1a0460b Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 13:08:30 -0700 Subject: [PATCH 10/11] fix: launcher warns loudly on insecure fallback, validates all certs Check all three cert files (ca.crt, tls.crt, tls.key) instead of just tls.crt. Print WARNING to stderr with specific missing files when falling back to insecure mode. Log success when mTLS is configured. Co-Authored-By: Claude Opus 4.6 (1M context) --- sandbox/launcher/main.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sandbox/launcher/main.go b/sandbox/launcher/main.go index 17f0abb..3defe2a 100644 --- a/sandbox/launcher/main.go +++ b/sandbox/launcher/main.go @@ -44,13 +44,21 @@ func parseConfig(path string) (*Config, error) { } func configureGateway(endpoint, mtlsDir, cli string) error { - certFile := filepath.Join(mtlsDir, "tls.crt") - if _, err := os.Stat(certFile); err != nil { - fmt.Println(" No mTLS certs, using insecure mode") + requiredCerts := []string{"ca.crt", "tls.crt", "tls.key"} + var missing []string + for _, name := range requiredCerts { + if _, err := os.Stat(filepath.Join(mtlsDir, name)); err != nil { + missing = append(missing, name) + } + } + if len(missing) > 0 { + fmt.Fprintf(os.Stderr, "WARNING: mTLS certs missing from %s: %v\n", mtlsDir, missing) + fmt.Fprintf(os.Stderr, "WARNING: falling back to INSECURE mode — gateway connection is NOT encrypted\n") os.Setenv("OPENSHELL_GATEWAY_ENDPOINT", endpoint) os.Setenv("OPENSHELL_GATEWAY_INSECURE", "true") return nil } + fmt.Println(" ✓ mTLS certs found — using encrypted connection") httpEndpoint := strings.Replace(endpoint, "https:", "http:", 1) From 62eae799712f5e726da1a89526ce7df001646eb6 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 5 Jun 2026 13:22:18 -0700 Subject: [PATCH 11/11] feat: add LAUNCHER_IMAGE env var override for Job spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matches existing SANDBOX_IMAGE pattern. Allows testing launcher changes locally without waiting for ghcr.io image push: LAUNCHER_IMAGE=quay.io/rcochran/openshell:launcher harness new --remote Verified: launcher mTLS cert validation working in OCP cluster logs: ✓ mTLS certs found — using encrypted connection Co-Authored-By: Claude Opus 4.6 (1M context) --- cmd/new.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/new.go b/cmd/new.go index 491b792..81fe8c4 100644 --- a/cmd/new.go +++ b/cmd/new.go @@ -174,7 +174,7 @@ func newRemote(harnessDir string, gw gateway.Gateway, profileName, sandboxName s "restartPolicy": "Never", "containers": []map[string]any{{ "name": "launcher", - "image": "ghcr.io/robbycochran/harness-openshell:launcher", + "image": envOr("LAUNCHER_IMAGE", "ghcr.io/robbycochran/harness-openshell:launcher"), "imagePullPolicy": "Always", "env": []map[string]any{ {"name": "GATEWAY_ENDPOINT", "value": "https://openshell.openshell.svc.cluster.local:8080"},