Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 114 additions & 0 deletions cmd/creds.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package cmd

import (
"context"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"

"github.com/robbycochran/harness-openshell/internal/k8s"
)

func ensureCreds(namespace string, force bool) error {
ctx := context.Background()
kc := k8s.New("", namespace)

if !kc.NamespaceExists(ctx, namespace) {
return fmt.Errorf("namespace '%s' not found — run: harness deploy --remote", namespace)
}

fmt.Println("=== Setting up cluster credentials ===")
fmt.Printf(" Namespace: %s\n", namespace)
fmt.Println()

// GWS credentials
fmt.Println("=== GWS ===")
if force && kc.SecretExists(ctx, "openshell-gws") {
kc.RunKubectl(ctx, "delete", "secret", "openshell-gws")
fmt.Println(" Deleted existing secret.")
}

if kc.SecretExists(ctx, "openshell-gws") {
fmt.Println(" openshell-gws: exists (use --force to recreate)")
} else {
if err := createGWSSecret(ctx, kc); err != nil {
fmt.Printf(" GWS: %v\n", err)
}
}

// Atlassian credentials
fmt.Println()
fmt.Println("=== Atlassian ===")
if force && kc.SecretExists(ctx, "openshell-atlassian") {
kc.RunKubectl(ctx, "delete", "secret", "openshell-atlassian")
fmt.Println(" Deleted existing secret.")
}

if kc.SecretExists(ctx, "openshell-atlassian") {
fmt.Println(" openshell-atlassian: exists (use --force to recreate)")
} else if jiraURL := os.Getenv("JIRA_URL"); jiraURL != "" {
jiraUser := os.Getenv("JIRA_USERNAME")
_, err := kc.RunKubectl(ctx, "create", "secret", "generic", "openshell-atlassian",
"--from-literal=JIRA_URL="+jiraURL,
"--from-literal=JIRA_USERNAME="+jiraUser)
if err != nil {
return fmt.Errorf("creating atlassian secret: %w", err)
}
fmt.Printf(" openshell-atlassian: created (%s)\n", jiraURL)
} else {
fmt.Println(" Atlassian: JIRA_URL not set (skipping)")
}

return nil
}

func createGWSSecret(ctx context.Context, kc *k8s.Client) error {
gwsPath, err := exec.LookPath("gws")
if err != nil {
fmt.Println(" GWS: not installed (skipping)")
return nil
}

check := exec.Command(gwsPath, "auth", "status")
check.Stdout = io.Discard
check.Stderr = io.Discard
if check.Run() != nil {
fmt.Println(" GWS: not authenticated (run 'gws auth login')")
return nil
}

tmpDir, err := os.MkdirTemp("", "harness-gws-")
if err != nil {
return err
}
defer os.RemoveAll(tmpDir)

credFile := filepath.Join(tmpDir, "credentials.json")
out, err := exec.Command(gwsPath, "auth", "export", "--unmasked").Output()
if err != nil {
fmt.Println(" GWS: export failed (skipping)")
return nil
}
os.WriteFile(credFile, out, 0o600)

args := []string{"create", "secret", "generic", "openshell-gws",
"--from-file=credentials.json=" + credFile}

gwsConfigDir := os.Getenv("GWS_CONFIG_DIR")
if gwsConfigDir == "" {
home, _ := os.UserHomeDir()
gwsConfigDir = filepath.Join(home, ".config", "gws")
}
clientSecret := filepath.Join(gwsConfigDir, "client_secret.json")
if _, err := os.Stat(clientSecret); err == nil {
args = append(args, "--from-file=client_secret.json="+clientSecret)
}

if _, err := kc.RunKubectl(ctx, args...); err != nil {
return fmt.Errorf("creating gws secret: %w", err)
}
fmt.Println(" openshell-gws: created")
return nil
}
203 changes: 194 additions & 9 deletions cmd/deploy.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package cmd

import (
"context"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/robbycochran/harness-openshell/internal/gateway"
"github.com/robbycochran/harness-openshell/internal/runner"
"github.com/robbycochran/harness-openshell/internal/k8s"
"github.com/robbycochran/harness-openshell/internal/preflight"
"github.com/robbycochran/harness-openshell/internal/status"
"github.com/spf13/cobra"
)
Expand All @@ -23,11 +28,8 @@ func NewDeployCmd(harnessDir, cli string) *cobra.Command {
Short: "Deploy or verify the gateway",
RunE: func(cmd *cobra.Command, args []string) error {
if remote {
scriptArgs := []string{"--remote"}
if kubeconfig != "" {
scriptArgs = append(scriptArgs, "--kubeconfig", kubeconfig)
}
return runner.RunScript(harnessDir, "deploy.sh", scriptArgs...)
gw := gateway.NewCLI(cli)
return deployRemote(harnessDir, gw, kubeconfig)
}
if local {
gw := gateway.NewCLI(cli)
Expand All @@ -44,14 +46,198 @@ func NewDeployCmd(harnessDir, cli string) *cobra.Command {
return cmd
}

func deployRemote(harnessDir string, gw gateway.Gateway, kubeconfig string) error {
ctx := context.Background()
namespace := os.Getenv("OPENSHELL_NAMESPACE")
if namespace == "" {
namespace = "openshell"
}

chartVersion := os.Getenv("OPENSHELL_CHART_VERSION")
if chartVersion == "" {
cfg, _ := preflight.LoadConfig(filepath.Join(harnessDir, "openshell.toml"))
if cfg != nil {
chartVersion = cfg.ChartVersion
}
}
if chartVersion == "" {
chartVersion = "0.0.55"
}
chartOCI := "oci://ghcr.io/nvidia/openshell/helm-chart"

fmt.Printf("OpenShell chart: %s\n", chartVersion)
if kubeconfig != "" {
fmt.Printf("KUBECONFIG: %s\n", kubeconfig)
}
fmt.Println()

kc := k8s.New(kubeconfig, namespace)
kcNoNS := k8s.New(kubeconfig, "")

// Step 1: Namespace
fmt.Println("=== Step 1: Creating namespace ===")
kcNoNS.RunKubectl(ctx, "create", "ns", namespace)
kcNoNS.RunKubectl(ctx, "label", "ns", namespace,
"pod-security.kubernetes.io/enforce=privileged",
"pod-security.kubernetes.io/warn=privileged",
"--overwrite")

// Step 2: Sandbox CRD
fmt.Println("=== Step 2: Installing Sandbox CRD ===")
kcNoNS.RunKubectlPassthrough(ctx, "apply", "-f",
"https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/manifest.yaml")

// Step 3: OpenShift SCCs
fmt.Println("=== Step 3: Granting OpenShift SCCs ===")
for _, sa := range []string{"openshell", "openshell-sandbox", "default"} {
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "privileged", "-z", sa, "-n", namespace)
}
kc.RunOC(ctx, "adm", "policy", "add-scc-to-user", "anyuid", "-z", "openshell", "-n", namespace)
kcNoNS.RunKubectl(ctx, "create", "clusterrolebinding", "agent-sandbox-admin",
"--clusterrole=cluster-admin",
"--serviceaccount=agent-sandbox-system:agent-sandbox-controller")

// RBAC for launcher
kc.ApplyYAML(ctx,
map[string]any{
"apiVersion": "v1",
"kind": "ServiceAccount",
"metadata": map[string]any{"name": "openshell-launcher", "namespace": namespace},
},
map[string]any{
"apiVersion": "rbac.authorization.k8s.io/v1",
"kind": "Role",
"metadata": map[string]any{"name": "openshell-launcher", "namespace": namespace},
"rules": []map[string]any{{
"apiGroups": []string{""},
"resources": []string{"configmaps", "secrets"},
"verbs": []string{"get", "list"},
}},
},
map[string]any{
"apiVersion": "rbac.authorization.k8s.io/v1",
"kind": "RoleBinding",
"metadata": map[string]any{"name": "openshell-launcher", "namespace": namespace},
"subjects": []map[string]any{{
"kind": "ServiceAccount",
"name": "openshell-launcher",
"namespace": namespace,
}},
"roleRef": map[string]any{
"kind": "Role",
"name": "openshell-launcher",
"apiGroup": "rbac.authorization.k8s.io",
},
},
)

// Step 4: Helm install
fmt.Println("=== Step 4: Deploying gateway via Helm ===")
sandboxImage := os.Getenv("SANDBOX_IMAGE")
if sandboxImage == "" {
sandboxImage = "quay.io/rcochran/openshell:sandbox"
}

appsDomain, err := kcNoNS.GetJSONPath(ctx, "ingresses.config.openshift.io/cluster", "{.spec.domain}")
if err != nil || appsDomain == "" {
return fmt.Errorf("could not determine OpenShift apps domain")
}
routeHost := fmt.Sprintf("gateway-openshell.%s", appsDomain)

helmArgs := []string{
"upgrade", "--install", "openshell", chartOCI,
"--version", chartVersion,
"--values", filepath.Join(harnessDir, "values-ocp.yaml"),
"--set", "server.sandboxImage=" + sandboxImage,
"--set", "pkiInitJob.serverDnsNames[0]=" + routeHost,
}
if ps := os.Getenv("PULL_SECRET"); ps != "" {
helmArgs = append(helmArgs, "--set", "imagePullSecrets[0].name="+ps)
}
if sps := os.Getenv("SANDBOX_PULL_SECRET"); sps != "" {
helmArgs = append(helmArgs, "--set", "server.sandboxImagePullSecrets[0].name="+sps)
}
kc.RunHelm(ctx, helmArgs...)

// Wait for gateway
fmt.Println("=== Waiting for gateway ===")
kc.RunKubectlPassthrough(ctx, "rollout", "status", "statefulset/openshell", "--timeout=300s")

// Step 5: Route
fmt.Println("=== Step 5: Creating OpenShift route ===")
if err := kc.RunKubectlQuiet(ctx, "get", "route", "gateway"); err != nil {
kc.ApplyYAML(ctx, map[string]any{
"apiVersion": "route.openshift.io/v1",
"kind": "Route",
"metadata": map[string]any{"name": "gateway", "namespace": namespace},
"spec": map[string]any{
"tls": map[string]any{"termination": "passthrough"},
"to": map[string]any{"kind": "Service", "name": "openshell"},
"port": map[string]any{"targetPort": "grpc"},
},
})
}
fmt.Printf(" Route: %s\n", routeHost)

// Step 6: CLI gateway config
fmt.Println("=== Step 6: Configuring CLI gateway ===")
gatewayName := os.Getenv("GATEWAY_NAME")
if gatewayName == "" {
gatewayName = "openshell-remote-ocp"
}
gatewayURL := fmt.Sprintf("https://%s:443", routeHost)

// Remove existing gateways for this host
existing, _ := gw.GatewayList()
for _, g := range existing {
if strings.Contains(g.Endpoint, routeHost) {
gw.GatewayRemove(g.Name)
}
}

gw.GatewayAdd(gatewayURL, gatewayName, true)

// Extract mTLS certs
home, _ := os.UserHomeDir()
mtlsDir := filepath.Join(home, ".config", "openshell", "gateways", gatewayName, "mtls")
for _, field := range []string{"ca.crt", "tls.crt", "tls.key"} {
data, err := kc.GetSecretField(ctx, "openshell-client-tls", field)
if err != nil {
return fmt.Errorf("extracting %s from openshell-client-tls: %w", field, err)
}
if err := os.WriteFile(filepath.Join(mtlsDir, field), data, 0o600); err != nil {
return fmt.Errorf("writing %s: %w", field, err)
}
}

gw.GatewaySelect(gatewayName)
status.OKf("%s registered (certs from cluster)", gatewayName)

// Wait for gateway to be reachable
fmt.Print(" Waiting for gateway...")
for i := range 30 {
if gw.InferenceGet() == nil {
fmt.Println(" ✓ reachable")
break
}
time.Sleep(2 * time.Second)
fmt.Print(".")
if i == 29 {
fmt.Println(" ✗ timed out (try: openshell inference get)")
}
}

fmt.Println()
fmt.Println("Done.")
return nil
}

func deployLocal(gw gateway.Gateway) error {
// CLI check
cliPath := gw.CLIPath()
if cliPath == "" {
return fmt.Errorf("openshell CLI not found. Install it first:\n curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/main/install.sh | sh")
}

// Podman check
status.Section("Container Runtime")
podmanPath, _ := exec.LookPath("podman")
if podmanPath == "" {
Expand All @@ -62,7 +248,6 @@ func deployLocal(gw gateway.Gateway) error {
out, _ := cmd.Output()
status.OKf("Podman: %s", strings.TrimSpace(string(out)))

// Find local gateway
status.Section("Gateway")
gateways, err := gw.GatewayList()
if err != nil {
Expand Down
Loading