Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion cmd/creds.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,9 @@ func createGWSSecret(ctx context.Context, kc *k8s.Client) error {
fmt.Println(" GWS: export failed (skipping)")
return nil
}
os.WriteFile(credFile, out, 0o600)
if err := os.WriteFile(credFile, out, 0o600); err != nil {
return fmt.Errorf("writing gws credentials: %w", err)
}

args := []string{"create", "secret", "generic", "openshell-gws",
"--from-file=credentials.json=" + credFile}
Expand Down
36 changes: 24 additions & 12 deletions cmd/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,20 +74,24 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) erro
kc := k8s.New(kubeconfig, namespace)
kcNoNS := k8s.New(kubeconfig, "")

// Step 1: Namespace
// Step 1: Namespace (idempotent — ignore AlreadyExists)
fmt.Println("=== Step 1: Creating namespace ===")
kcNoNS.RunKubectl(ctx, "create", "ns", namespace)
kcNoNS.RunKubectl(ctx, "label", "ns", namespace,
if _, err := kcNoNS.RunKubectl(ctx, "label", "ns", namespace,
"pod-security.kubernetes.io/enforce=privileged",
"pod-security.kubernetes.io/warn=privileged",
"--overwrite")
"--overwrite"); err != nil {
return fmt.Errorf("labeling namespace: %w", err)
}

// Step 2: Sandbox CRD
fmt.Println("=== Step 2: Installing Sandbox CRD ===")
kcNoNS.RunKubectlPassthrough(ctx, "apply", "-f",
"https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/manifest.yaml")
if err := kcNoNS.RunKubectlPassthrough(ctx, "apply", "-f",
"https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/manifest.yaml"); err != nil {
return fmt.Errorf("installing sandbox CRD: %w", err)
}

// Step 3: OpenShift SCCs
// Step 3: OpenShift SCCs (best-effort — oc may not exist on non-OpenShift)
fmt.Println("=== Step 3: Granting OpenShift SCCs ===")
for _, sa := range []string{"openshell", "openshell-sandbox", "default"} {
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "privileged", "-z", sa, "-n", namespace)
Expand All @@ -98,7 +102,7 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) erro
"--serviceaccount=agent-sandbox-system:agent-sandbox-controller")

// RBAC for launcher
kc.ApplyYAML(ctx,
if err := kc.ApplyYAML(ctx,
map[string]any{
"apiVersion": "v1",
"kind": "ServiceAccount",
Expand Down Expand Up @@ -129,7 +133,9 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) erro
"apiGroup": "rbac.authorization.k8s.io",
},
},
)
); err != nil {
return fmt.Errorf("applying launcher RBAC: %w", err)
}

// Step 4: Helm install
fmt.Println("=== Step 4: Deploying gateway via Helm ===")
Expand All @@ -140,7 +146,7 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) erro

appsDomain, err := kcNoNS.GetJSONPath(ctx, "ingresses.config.openshift.io/cluster", "{.spec.domain}")
if err != nil || appsDomain == "" {
return fmt.Errorf("could not determine OpenShift apps domain")
return fmt.Errorf("could not determine OpenShift apps domain — is this an OpenShift cluster? (kubectl get ingresses.config.openshift.io cluster)")
}
routeHost := fmt.Sprintf("gateway-openshell.%s", appsDomain)

Expand All @@ -157,11 +163,15 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) erro
if sps := os.Getenv("SANDBOX_PULL_SECRET"); sps != "" {
helmArgs = append(helmArgs, "--set", "server.sandboxImagePullSecrets[0].name="+sps)
}
kc.RunHelm(ctx, helmArgs...)
if _, err := kc.RunHelm(ctx, helmArgs...); err != nil {
return fmt.Errorf("helm install failed: %w", err)
}

// Wait for gateway
fmt.Println("=== Waiting for gateway ===")
kc.RunKubectlPassthrough(ctx, "rollout", "status", "statefulset/openshell", "--timeout=300s")
if err := kc.RunKubectlPassthrough(ctx, "rollout", "status", "statefulset/openshell", "--timeout=300s"); err != nil {
return fmt.Errorf("gateway rollout failed: %w", err)
}

// Step 5: Route
fmt.Println("=== Step 5: Creating OpenShift route ===")
Expand Down Expand Up @@ -195,7 +205,9 @@ func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) erro
}
}

gw.GatewayAdd(gatewayURL, gatewayName, true)
if err := gw.GatewayAdd(gatewayURL, gatewayName, true); err != nil {
return fmt.Errorf("registering gateway %s: %w", gatewayName, err)
}

// Extract mTLS certs
home, _ := os.UserHomeDir()
Expand Down
42 changes: 25 additions & 17 deletions cmd/new.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,12 @@ func newRemote(harnessDir string, gw gateway.Gateway, profileName, sandboxName s
if err != nil {
return fmt.Errorf("creating config configmap: %w", err)
}
kc.RunKubectlOpts(ctx, k8s.KubectlOpts{
if _, err := kc.RunKubectlOpts(ctx, k8s.KubectlOpts{
Args: []string{"apply", "-f", "-"},
Stdin: strings.NewReader(out),
})
}); err != nil {
return fmt.Errorf("applying config configmap: %w", err)
}

// 2. ConfigMap from env (conditional)
envContent := cfg.BuildSandboxEnv()
Expand All @@ -135,14 +137,16 @@ func newRemote(harnessDir string, gw gateway.Gateway, profileName, sandboxName s
"--from-literal=sandbox.env="+envContent,
"--dry-run=client", "-o", "yaml")
if err == nil {
kc.RunKubectlOpts(ctx, k8s.KubectlOpts{
if _, err := kc.RunKubectlOpts(ctx, k8s.KubectlOpts{
Args: []string{"apply", "-f", "-"},
Stdin: strings.NewReader(out),
})
}); err != nil {
return fmt.Errorf("applying env configmap: %w", err)
}
}
}

// 3. Clean up old job
// 3. Clean up old job (best-effort — may not exist)
jobName := "sandbox-" + cfg.Name
kc.RunKubectl(ctx, "delete", "job", jobName, "--force", "--grace-period=0")
kc.RunKubectl(ctx, "delete", "pod", "-l", "job-name="+jobName, "--force", "--grace-period=0")
Expand Down Expand Up @@ -200,9 +204,11 @@ func newRemote(harnessDir string, gw gateway.Gateway, profileName, sandboxName s
logCmd.Stderr = os.Stderr
logCmd.Start()

// 7. Poll job status
for {
jobStatus, _ := kc.RunKubectl(ctx, "get", "job", jobName,
// 7. Poll job status (10 min timeout)
var jobStatus string
deadline := time.Now().Add(10 * time.Minute)
for time.Now().Before(deadline) {
jobStatus, _ = kc.RunKubectl(ctx, "get", "job", jobName,
"-o", "jsonpath={.status.conditions[0].type}")
if jobStatus == "Complete" || jobStatus == "Failed" || jobStatus == "SuccessCriteriaMet" {
break
Expand All @@ -216,14 +222,14 @@ func newRemote(harnessDir string, gw gateway.Gateway, profileName, sandboxName s
}

fmt.Println()
jobStatus, _ := kc.RunKubectl(ctx, "get", "job", jobName,
"-o", "jsonpath={.status.conditions[0].type}")
if jobStatus == "Complete" || jobStatus == "SuccessCriteriaMet" {
fmt.Printf("Sandbox ready. Connect with: harness connect %s\n", cfg.Name)
} else {
fmt.Println("Launcher failed.")
return nil
}
return nil
if jobStatus == "" {
return fmt.Errorf("launcher job timed out — check: kubectl logs -n %s -l job-name=%s", namespace, jobName)
}
return fmt.Errorf("launcher job failed (status: %s) — check: kubectl logs -n %s -l job-name=%s", jobStatus, namespace, jobName)
}

func newLocal(opts newLocalOpts) error {
Expand Down Expand Up @@ -280,7 +286,9 @@ func newLocal(opts newLocalOpts) error {

// 5. Stage files
harnessUploadDir := "/tmp/openshell"
os.RemoveAll(harnessUploadDir)
if err := os.RemoveAll(harnessUploadDir); err != nil {
return fmt.Errorf("cleaning staging dir: %w", err)
}
if err := profile.StageHarnessDir(cfg, harnessUploadDir); err != nil {
return fmt.Errorf("staging files: %w", err)
}
Expand Down Expand Up @@ -311,11 +319,11 @@ func newLocal(opts newLocalOpts) error {
return nil
}

fmt.Printf(" Attempt %d failed (supervisor race), retrying in 5s...\n", attempt)
gw.SandboxDelete(cfg.Name)
fmt.Printf(" Attempt %d failed: %v, retrying in 5s...\n", attempt, err)
gw.SandboxDelete(cfg.Name) // best-effort cleanup

if attempt == 5 {
return fmt.Errorf("failed after 5 attempts")
return fmt.Errorf("sandbox create failed after 5 attempts: %w", err)
}
time.Sleep(opts.retrySleep)
}
Expand Down
4 changes: 3 additions & 1 deletion internal/profile/profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,9 @@ func stageGWSCreds(harnessDir string) error {
}
clientSecret := filepath.Join(gwsConfigDir, "client_secret.json")
if data, err := os.ReadFile(clientSecret); err == nil {
os.WriteFile(filepath.Join(harnessDir, "client_secret.json"), data, 0o600)
if err := os.WriteFile(filepath.Join(harnessDir, "client_secret.json"), data, 0o600); err != nil {
return fmt.Errorf("writing client_secret.json: %w", err)
}
}

fmt.Println(" GWS: exported")
Expand Down
4 changes: 4 additions & 0 deletions test/test-flow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,10 @@ test_ocp() {
echo "=== test-flow: ocp ($mode) [$IMPL] ==="

if $REUSE_GATEWAY; then
# Ensure OCP gateway is selected (error scenarios may have switched to local)
OCP_GW=$("$CLI" gateway list 2>/dev/null | strip_ansi | awk '/-remote-/ {gsub(/^\*/, ""); print $1; exit}')
[[ -n "$OCP_GW" ]] && "$CLI" gateway select "$OCP_GW" 2>/dev/null || true

step "teardown sandboxes+providers" "$HARNESS" teardown --sandboxes --providers
# Deploy only if gateway is not reachable
if ! "$CLI" inference get &>/dev/null; then
Expand Down