feat: test-flow.sh + make test for end-to-end validation

Makefile

@@ -19,7 +19,7 @@ SUPERVISOR_IMAGE := $(REGISTRY):supervisor
 
 .PHONY: all images push sandbox launcher gateway supervisor \
         cli cli-gateway cli-supervisor cli-launcher \
-        clean help
+        test test-podman test-ocp clean help
 
 all: images push
 
@@ -124,6 +124,20 @@ $(OPENSHELL_REPO)/target/x86_64-unknown-linux-musl/release/openshell-sandbox:
 	  -p openshell-sandbox
 	@echo "Cross-compiled: openshell-sandbox (linux/amd64 musl)"
 
+## ── Test targets ─────────────────────────────────────────────────────
+
+## Build + push sandbox and launcher, then run full tests on both platforms
+test: sandbox push-launcher
+	./test-flow.sh all --full
+
+## Build + push sandbox and launcher, then run full podman test
+test-podman: sandbox push-launcher
+	./test-flow.sh podman --full
+
+## Build + push sandbox and launcher, then run full OCP test
+test-ocp: sandbox push-launcher
+	./test-flow.sh ocp --full
+
 ## ── Convenience targets ───────────────────────────────────────────────
 
 ## Build everything from source (slow — cross-compiles all binaries)

sandbox-podman.sh

@@ -6,8 +6,10 @@
 #   ./setup-providers.sh      # register providers
 #
 # Usage:
-#   ./sandbox-podman.sh              # uses agents/default.toml
-#   ./sandbox-podman.sh research     # uses agents/research.toml
+#   ./sandbox-podman.sh                        # interactive, uses agents/default.toml
+#   ./sandbox-podman.sh research               # uses agents/research.toml
+#   ./sandbox-podman.sh --name my-agent        # named sandbox
+#   ./sandbox-podman.sh --no-tty --name test   # non-interactive (for testing)
 #
 # To reconnect: openshell sandbox connect <name>
 # To delete:    openshell sandbox delete <name>
@@ -18,7 +20,19 @@ source "$SCRIPT_DIR/lib/common.sh"
 require_cli
 require_local_gateway
 
-AGENT_NAME="${1:-default}"
+# Parse args
+AGENT_NAME="default"
+SANDBOX_NAME=""
+TTY_FLAG="--tty"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --name) SANDBOX_NAME="$2"; shift 2 ;;
+    --no-tty) TTY_FLAG="--no-tty"; shift ;;
+    *) AGENT_NAME="$1"; shift ;;
+  esac
+done
+
 AGENT_FILE="$SCRIPT_DIR/agents/${AGENT_NAME}.toml"
 [[ -f "$AGENT_FILE" ]] || { echo "ERROR: $AGENT_FILE not found."; exit 1; }
 
@@ -48,17 +62,15 @@ for name in $SANDBOX_PROVIDERS; do
   fi
 done
 
-# Stage files for upload to /sandbox/.config/openshell/ (same as OCP launcher)
+# Stage files for upload to /sandbox/.config/openshell/
 HARNESS_DIR="/tmp/openshell"
 mkdir -p "$HARNESS_DIR"
 
-# Sandbox env vars from agent config
 if [[ -n "$SANDBOX_ENV" ]]; then
   echo "$SANDBOX_ENV" > "$HARNESS_DIR/sandbox.env"
   echo "  Env: $(wc -l < "$HARNESS_DIR/sandbox.env") vars"
 fi
 
-# GWS credentials
 if command -v gws &>/dev/null && gws auth status &>/dev/null; then
   gws auth export --unmasked > "$HARNESS_DIR/credentials.json" 2>/dev/null
   cp ~/.config/gws/client_secret.json "$HARNESS_DIR/client_secret.json" 2>/dev/null || true
@@ -67,25 +79,38 @@ else
   echo "  GWS: not configured (skipping)"
 fi
 
-# Image flag
+# Build flags
 FROM_FLAGS=()
-if [[ -n "$SANDBOX_IMAGE" ]]; then
-  FROM_FLAGS=(--from "$SANDBOX_IMAGE")
+[[ -n "$SANDBOX_IMAGE" ]] && FROM_FLAGS=(--from "$SANDBOX_IMAGE")
+
+NAME_FLAGS=()
+[[ -n "$SANDBOX_NAME" ]] && NAME_FLAGS=(--name "$SANDBOX_NAME")
+
+# Command depends on tty mode
+if [[ "$TTY_FLAG" == "--tty" ]]; then
+  CMD=(-- bash -c ". /sandbox/startup.sh && exec $SANDBOX_COMMAND")
+else
+  CMD=(-- bash /sandbox/startup.sh)
 fi
 
 echo ""
 echo "=== Creating sandbox ==="
-for attempt in 1 2 3; do
+for attempt in 1 2 3 4 5; do
   "$CLI" sandbox create \
-    --tty \
+    $TTY_FLAG \
+    ${NAME_FLAGS[@]+"${NAME_FLAGS[@]}"} \
     ${FROM_FLAGS[@]+"${FROM_FLAGS[@]}"} \
     ${PROVIDER_FLAGS[@]+"${PROVIDER_FLAGS[@]}"} \
     --upload "$HARNESS_DIR:/sandbox/.config" --no-git-ignore \
-    -- bash -c '. /sandbox/startup.sh && exec '"$SANDBOX_COMMAND" \
+    "${CMD[@]}" \
     && exit 0
   echo "  Attempt $attempt failed (supervisor race), retrying in 5s..."
-  "$CLI" sandbox delete "$("$CLI" sandbox list 2>/dev/null | awk 'NR==2{print $1}')" 2>/dev/null || true
+  if [[ -n "$SANDBOX_NAME" ]]; then
+    "$CLI" sandbox delete "$SANDBOX_NAME" 2>/dev/null || true
+  else
+    "$CLI" sandbox delete "$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk 'NR==2{print $1}')" 2>/dev/null || true
+  fi
   sleep 5
 done
-echo "ERROR: Failed after 3 attempts."
+echo "ERROR: Failed after 5 attempts."
 exit 1

test-flow.sh

@@ -0,0 +1,187 @@
+#!/usr/bin/env bash
+# End-to-end validation for podman and OCP flows.
+#
+# Usage:
+#   ./test-flow.sh podman          # quick: deploy + providers + teardown
+#   ./test-flow.sh podman --full   # full: + sandbox + verify integrations
+#   ./test-flow.sh ocp             # quick: deploy + creds + providers + teardown
+#   ./test-flow.sh ocp --full      # full: + sandbox + verify integrations
+#   ./test-flow.sh all             # quick for both
+#   ./test-flow.sh all --full      # full for both
+set -uo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+CLI="${OPENSHELL_CLI:-openshell}"
+
+TARGET="${1:-}"
+FULL=false
+[[ "${2:-}" == "--full" || "${3:-}" == "--full" ]] && FULL=true
+
+if [[ -z "$TARGET" ]]; then
+  echo "Usage: $0 <podman|ocp|all> [--full]"
+  exit 1
+fi
+
+# ── Helpers ──────────────────────────────────────────────────────────
+
+PASS=0
+FAIL=0
+TOTAL_START=$(date +%s)
+
+step() {
+  local label="$1"; shift
+  local start=$(date +%s)
+  if "$@" &>/dev/null; then
+    local elapsed=$(( $(date +%s) - start ))
+    printf "  ✓ %-35s (%ds)\n" "$label" "$elapsed"
+    ((PASS++))
+  else
+    local elapsed=$(( $(date +%s) - start ))
+    printf "  ✗ %-35s (%ds)\n" "$label" "$elapsed"
+    ((FAIL++))
+  fi
+}
+
+step_output() {
+  local label="$1"; shift
+  local start=$(date +%s)
+  local out
+  out=$("$@" 2>&1)
+  local rc=$?
+  local elapsed=$(( $(date +%s) - start ))
+  if [[ $rc -eq 0 ]]; then
+    printf "  ✓ %-35s (%ds)\n" "$label" "$elapsed"
+    ((PASS++))
+  else
+    printf "  ✗ %-35s (%ds)\n" "$label" "$elapsed"
+    echo "    $out" | head -3
+    ((FAIL++))
+  fi
+}
+
+check_providers() {
+  local count
+  count=$("$CLI" provider list 2>/dev/null | awk 'NR>1' | wc -l | tr -d ' ')
+  if [[ "$count" -gt 0 ]]; then
+    printf "  ✓ %-35s (%s)\n" "providers registered" "${count} providers"
+    ((PASS++))
+  else
+    printf "  ✗ %-35s\n" "providers registered (0)"
+    ((FAIL++))
+  fi
+}
+
+sandbox_verify() {
+  local name="$1"
+  # Check sandbox is ready
+  local phase
+  phase=$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk -v n="$name" '$1==n {print $NF}')
+  if [[ "$phase" != "Ready" ]]; then
+    printf "  ✗ %-35s\n" "sandbox ready"
+    ((FAIL++))
+    return
+  fi
+  printf "  ✓ %-35s\n" "sandbox ready"
+  ((PASS++))
+
+  # Check env vars
+  step "sandbox: env vars" "$CLI" sandbox exec --name "$name" -- bash -c 'test -n "$ANTHROPIC_BASE_URL"'
+
+  # Check GWS credentials
+  step "sandbox: gws creds" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.config/openshell/credentials.json
+
+  # Check MCP config
+  step "sandbox: mcp config" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.mcp.json
+
+  # Check Claude responds
+  step_output "sandbox: claude responds" "$CLI" sandbox exec --name "$name" -- bash -c 'echo "respond with ok" | claude --bare --print 2>&1 | head -1'
+}
+
+summary() {
+  local total=$(( PASS + FAIL ))
+  local elapsed=$(( $(date +%s) - TOTAL_START ))
+  echo ""
+  if [[ $FAIL -eq 0 ]]; then
+    echo "${PASS}/${total} passed (${elapsed}s)"
+  else
+    echo "${PASS}/${total} passed, ${FAIL} failed (${elapsed}s)"
+  fi
+}
+
+# ── Podman flow ──────────────────────────────────────────────────────
+
+test_podman() {
+  local mode="quick"
+  $FULL && mode="full"
+  echo "=== test-flow: podman ($mode) ==="
+
+  step "teardown" "$SCRIPT_DIR/teardown.sh" --sandboxes --providers
+  step "deploy" "$SCRIPT_DIR/deploy-podman.sh"
+  step "setup providers" "$SCRIPT_DIR/setup-providers.sh"
+  step "gateway reachable" "$CLI" inference get
+  check_providers
+
+  if $FULL; then
+    local sandbox_name="test-agent"
+    step_output "sandbox create" "$SCRIPT_DIR/sandbox-podman.sh" --name "$sandbox_name" --no-tty
+    sandbox_verify "$sandbox_name"
+    step "sandbox delete" "$CLI" sandbox delete "$sandbox_name"
+  fi
+
+  step "teardown (clean)" "$SCRIPT_DIR/teardown.sh" --sandboxes --providers
+}
+
+# ── OCP flow ─────────────────────────────────────────────────────────
+
+test_ocp() {
+  local mode="quick"
+  $FULL && mode="full"
+  echo "=== test-flow: ocp ($mode) ==="
+
+  step "teardown" "$SCRIPT_DIR/teardown.sh"
+  step "deploy" "$SCRIPT_DIR/deploy-ocp.sh"
+  step "setup creds" "$SCRIPT_DIR/setup-creds.sh"
+  step "setup providers" "$SCRIPT_DIR/setup-providers.sh"
+  step "gateway reachable" "$CLI" inference get
+  check_providers
+
+  if $FULL; then
+    step_output "sandbox create" "$SCRIPT_DIR/sandbox-ocp.sh"
+    local sandbox_name="agent"
+
+    # Wait for ready
+    for i in $(seq 1 30); do
+      local phase=$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk -v n="$sandbox_name" '$1==n {print $NF}')
+      [[ "$phase" == "Ready" ]] && break
+      sleep 2
+    done
+    sandbox_verify "$sandbox_name"
+    step "sandbox delete" "$CLI" sandbox delete "$sandbox_name"
+  fi
+
+  step "teardown (clean)" "$SCRIPT_DIR/teardown.sh"
+}
+
+# ── Main ─────────────────────────────────────────────────────────────
+
+case "$TARGET" in
+  podman)
+    test_podman
+    ;;
+  ocp)
+    test_ocp
+    ;;
+  all)
+    test_podman
+    echo ""
+    test_ocp
+    ;;
+  *)
+    echo "Unknown target: $TARGET"
+    echo "Usage: $0 <podman|ocp|all> [--full]"
+    exit 1
+    ;;
+esac
+
+summary
+exit $FAIL