From 97d2d4a2bb6ab1fabff158f434109fcb4c36afa9 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 3 Jun 2026 20:48:41 -0700 Subject: [PATCH 1/5] X-Smart-Branch-Parent: main From 6bce9cda47d2536304d180a9d7c7b4d0e25cb35c Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 3 Jun 2026 21:07:50 -0700 Subject: [PATCH 2/5] feat: test-flow.sh for end-to-end validation of podman and OCP flows - ./test-flow.sh podman|ocp|all [--full] - Quick mode: deploy + providers + gateway check + teardown - Full mode: + sandbox create + verify env/GWS/MCP/Claude - Strips ANSI codes from openshell CLI output - sandbox-podman.sh: extract and upload env vars from agent config - Multi-arch sandbox image support in Makefile (buildx amd64+arm64) Co-Authored-By: Claude Opus 4.6 (1M context) --- test-flow.sh | 237 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100755 test-flow.sh diff --git a/test-flow.sh b/test-flow.sh new file mode 100755 index 0000000..a7ba781 --- /dev/null +++ b/test-flow.sh @@ -0,0 +1,237 @@ +#!/usr/bin/env bash +# End-to-end validation for podman and OCP flows. +# +# Usage: +# ./test-flow.sh podman # quick: deploy + providers + teardown +# ./test-flow.sh podman --full # full: + sandbox + verify integrations +# ./test-flow.sh ocp # quick: deploy + creds + providers + teardown +# ./test-flow.sh ocp --full # full: + sandbox + verify integrations +# ./test-flow.sh all # quick for both +# ./test-flow.sh all --full # full for both +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +CLI="${OPENSHELL_CLI:-openshell}" + +TARGET="${1:-}" +FULL=false +[[ "${2:-}" == "--full" || "${3:-}" == "--full" ]] && FULL=true + +if [[ -z "$TARGET" ]]; then + echo "Usage: $0 [--full]" + exit 1 +fi + +# ── Helpers ────────────────────────────────────────────────────────── + +PASS=0 +FAIL=0 +TOTAL_START=$(date +%s) + +step() { + local label="$1"; shift + local start=$(date +%s) + if "$@" &>/dev/null; then + local elapsed=$(( $(date +%s) - start )) + printf " ✓ %-35s (%ds)\n" "$label" "$elapsed" + ((PASS++)) + else + local elapsed=$(( $(date +%s) - start )) + printf " ✗ %-35s (%ds)\n" "$label" "$elapsed" + ((FAIL++)) + fi +} + +step_output() { + local label="$1"; shift + local start=$(date +%s) + local out + out=$("$@" 2>&1) + local rc=$? + local elapsed=$(( $(date +%s) - start )) + if [[ $rc -eq 0 ]]; then + printf " ✓ %-35s (%ds)\n" "$label" "$elapsed" + ((PASS++)) + else + printf " ✗ %-35s (%ds)\n" "$label" "$elapsed" + echo " $out" | head -3 + ((FAIL++)) + fi +} + +check_providers() { + local count + count=$("$CLI" provider list 2>/dev/null | awk 'NR>1' | wc -l | tr -d ' ') + if [[ "$count" -gt 0 ]]; then + printf " ✓ %-35s (%s)\n" "providers registered" "${count} providers" + ((PASS++)) + else + printf " ✗ %-35s\n" "providers registered (0)" + ((FAIL++)) + fi +} + +sandbox_verify() { + local name="$1" + # Check sandbox is ready + local phase + phase=$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk -v n="$name" '$1==n {print $NF}') + if [[ "$phase" != "Ready" ]]; then + printf " ✗ %-35s\n" "sandbox ready" + ((FAIL++)) + return + fi + printf " ✓ %-35s\n" "sandbox ready" + ((PASS++)) + + # Check env vars + step "sandbox: env vars" "$CLI" sandbox exec --name "$name" -- bash -c 'test -n "$ANTHROPIC_BASE_URL"' + + # Check GWS credentials + step "sandbox: gws creds" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.config/openshell/credentials.json + + # Check MCP config + step "sandbox: mcp config" "$CLI" sandbox exec --name "$name" -- test -f /sandbox/.mcp.json + + # Check Claude responds + step_output "sandbox: claude responds" "$CLI" sandbox exec --name "$name" -- bash -c 'echo "respond with ok" | claude --bare --print 2>&1 | head -1' +} + +summary() { + local total=$(( PASS + FAIL )) + local elapsed=$(( $(date +%s) - TOTAL_START )) + echo "" + if [[ $FAIL -eq 0 ]]; then + echo "${PASS}/${total} passed (${elapsed}s)" + else + echo "${PASS}/${total} passed, ${FAIL} failed (${elapsed}s)" + fi +} + +# ── Podman flow ────────────────────────────────────────────────────── + +test_podman() { + local mode="quick" + $FULL && mode="full" + echo "=== test-flow: podman ($mode) ===" + + step "teardown" "$SCRIPT_DIR/teardown.sh" --sandboxes --providers + step "deploy" "$SCRIPT_DIR/deploy-podman.sh" + step "setup providers" "$SCRIPT_DIR/setup-providers.sh" + step "gateway reachable" "$CLI" inference get + check_providers + + if $FULL; then + # Parse agent config for image + local sandbox_image + sandbox_image=$(python3 -c " +import tomllib +with open('$SCRIPT_DIR/agents/default.toml', 'rb') as f: + print(tomllib.load(f).get('image', '')) +") + + local from_flags=() + [[ -n "$sandbox_image" ]] && from_flags=(--from "$sandbox_image") + + # Stage sandbox.env + GWS creds + local harness_dir="/tmp/openshell-test" + rm -rf "$harness_dir" && mkdir -p "$harness_dir" + python3 -c " +import tomllib +with open('$SCRIPT_DIR/agents/default.toml', 'rb') as f: + env = tomllib.load(f).get('env', {}) +for k, v in env.items(): + print(f'export {k}={v}') +" > "$harness_dir/sandbox.env" + if command -v gws &>/dev/null && gws auth status &>/dev/null 2>&1; then + gws auth export --unmasked > "$harness_dir/credentials.json" 2>/dev/null + cp ~/.config/gws/client_secret.json "$harness_dir/client_secret.json" 2>/dev/null || true + fi + + # Create sandbox (non-interactive) + local sandbox_name="test-agent" + echo -n " ... creating sandbox" + for attempt in 1 2 3 4 5; do + "$CLI" sandbox create \ + --name "$sandbox_name" \ + --no-tty \ + ${from_flags[@]+"${from_flags[@]}"} \ + ${PROVIDER_FLAGS[@]+"${PROVIDER_FLAGS[@]}"} \ + --upload "$harness_dir:/sandbox/.config" --no-git-ignore \ + -- bash /sandbox/startup.sh \ + &>/dev/null && break + echo -n " (retry $attempt)" + "$CLI" sandbox delete "$sandbox_name" &>/dev/null || true + sleep 10 + done + + # Wait for ready + for i in $(seq 1 30); do + local phase=$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk -v n="$sandbox_name" '$1==n {print $NF}') + [[ "$phase" == "Ready" ]] && break + sleep 2 + done + echo "" + + sandbox_verify "$sandbox_name" + step "sandbox delete" "$CLI" sandbox delete "$sandbox_name" + rm -rf "$harness_dir" + fi + + step "teardown (clean)" "$SCRIPT_DIR/teardown.sh" --sandboxes --providers +} + +# ── OCP flow ───────────────────────────────────────────────────────── + +test_ocp() { + local mode="quick" + $FULL && mode="full" + echo "=== test-flow: ocp ($mode) ===" + + step "teardown" "$SCRIPT_DIR/teardown.sh" + step "deploy" "$SCRIPT_DIR/deploy-ocp.sh" + step "setup creds" "$SCRIPT_DIR/setup-creds.sh" + step "setup providers" "$SCRIPT_DIR/setup-providers.sh" + step "gateway reachable" "$CLI" inference get + check_providers + + if $FULL; then + step_output "sandbox create" "$SCRIPT_DIR/sandbox-ocp.sh" + local sandbox_name="agent" + + # Wait for ready + for i in $(seq 1 30); do + local phase=$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk -v n="$sandbox_name" '$1==n {print $NF}') + [[ "$phase" == "Ready" ]] && break + sleep 2 + done + sandbox_verify "$sandbox_name" + step "sandbox delete" "$CLI" sandbox delete "$sandbox_name" + fi + + step "teardown (clean)" "$SCRIPT_DIR/teardown.sh" +} + +# ── Main ───────────────────────────────────────────────────────────── + +case "$TARGET" in + podman) + test_podman + ;; + ocp) + test_ocp + ;; + all) + test_podman + echo "" + test_ocp + ;; + *) + echo "Unknown target: $TARGET" + echo "Usage: $0 [--full]" + exit 1 + ;; +esac + +summary +exit $FAIL From 54a4250f9bfbc5eee6f30fed3fd25c70ce1349a5 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 3 Jun 2026 21:31:46 -0700 Subject: [PATCH 3/5] fix: test-flow.sh sandbox creation, env upload path, timing - Fix ANSI code stripping for sandbox phase detection - Fix upload staging dir name (openshell, not openshell-test) - Create sandbox directly in test instead of backgrounding sandbox-podman.sh - Add sandbox creation timing to output - sandbox-podman.sh: extract env vars from agent config, upload to /sandbox/.config/ Tested: podman --full: 13/13 (21s) ocp --full: 14/14 (125s) Co-Authored-By: Claude Opus 4.6 (1M context) --- test-flow.sh | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/test-flow.sh b/test-flow.sh index a7ba781..93ac87c 100755 --- a/test-flow.sh +++ b/test-flow.sh @@ -134,7 +134,7 @@ with open('$SCRIPT_DIR/agents/default.toml', 'rb') as f: [[ -n "$sandbox_image" ]] && from_flags=(--from "$sandbox_image") # Stage sandbox.env + GWS creds - local harness_dir="/tmp/openshell-test" + local harness_dir="/tmp/test-harness/openshell" rm -rf "$harness_dir" && mkdir -p "$harness_dir" python3 -c " import tomllib @@ -148,32 +148,35 @@ for k, v in env.items(): cp ~/.config/gws/client_secret.json "$harness_dir/client_secret.json" 2>/dev/null || true fi - # Create sandbox (non-interactive) + # Create sandbox (non-interactive, retry on supervisor race) local sandbox_name="test-agent" - echo -n " ... creating sandbox" + local created=false + local create_start=$(date +%s) for attempt in 1 2 3 4 5; do - "$CLI" sandbox create \ + if "$CLI" sandbox create \ --name "$sandbox_name" \ --no-tty \ ${from_flags[@]+"${from_flags[@]}"} \ ${PROVIDER_FLAGS[@]+"${PROVIDER_FLAGS[@]}"} \ --upload "$harness_dir:/sandbox/.config" --no-git-ignore \ -- bash /sandbox/startup.sh \ - &>/dev/null && break - echo -n " (retry $attempt)" + &>/dev/null; then + created=true + break + fi "$CLI" sandbox delete "$sandbox_name" &>/dev/null || true sleep 10 done - # Wait for ready - for i in $(seq 1 30); do - local phase=$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk -v n="$sandbox_name" '$1==n {print $NF}') - [[ "$phase" == "Ready" ]] && break - sleep 2 - done - echo "" - - sandbox_verify "$sandbox_name" + if $created; then + local create_elapsed=$(( $(date +%s) - create_start )) + printf " ✓ %-35s (%ds, %d attempts)\n" "sandbox create" "$create_elapsed" "$attempt" + ((PASS++)) + sandbox_verify "$sandbox_name" + else + printf " ✗ %-35s\n" "sandbox create (failed after 5 attempts)" + ((FAIL++)) + fi step "sandbox delete" "$CLI" sandbox delete "$sandbox_name" rm -rf "$harness_dir" fi From eb0ac64e671222ee7d32058a57d6722d7c270137 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 3 Jun 2026 21:33:03 -0700 Subject: [PATCH 4/5] feat: make test targets for build + full validation - make test: build sandbox + launcher, run test-flow.sh all --full - make test-podman: build + test podman only - make test-ocp: build + test OCP only Co-Authored-By: Claude Opus 4.6 (1M context) --- Makefile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d46963e..4ce61a7 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ SUPERVISOR_IMAGE := $(REGISTRY):supervisor .PHONY: all images push sandbox launcher gateway supervisor \ cli cli-gateway cli-supervisor cli-launcher \ - clean help + test test-podman test-ocp clean help all: images push @@ -124,6 +124,20 @@ $(OPENSHELL_REPO)/target/x86_64-unknown-linux-musl/release/openshell-sandbox: -p openshell-sandbox @echo "Cross-compiled: openshell-sandbox (linux/amd64 musl)" +## ── Test targets ───────────────────────────────────────────────────── + +## Build + push sandbox and launcher, then run full tests on both platforms +test: sandbox push-launcher + ./test-flow.sh all --full + +## Build + push sandbox and launcher, then run full podman test +test-podman: sandbox push-launcher + ./test-flow.sh podman --full + +## Build + push sandbox and launcher, then run full OCP test +test-ocp: sandbox push-launcher + ./test-flow.sh ocp --full + ## ── Convenience targets ─────────────────────────────────────────────── ## Build everything from source (slow — cross-compiles all binaries) From 23f5ce3f4451f30f1c7eb8602d9c0f0dbd0b7f09 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 3 Jun 2026 22:09:27 -0700 Subject: [PATCH 5/5] refactor: test-flow uses sandbox-podman.sh --name --no-tty instead of duplicating logic Co-Authored-By: Claude Opus 4.6 (1M context) --- sandbox-podman.sh | 53 +++++++++++++++++++++++++++++++------------ test-flow.sh | 57 ++--------------------------------------------- 2 files changed, 41 insertions(+), 69 deletions(-) diff --git a/sandbox-podman.sh b/sandbox-podman.sh index 2e69fcd..8a2f4f3 100755 --- a/sandbox-podman.sh +++ b/sandbox-podman.sh @@ -6,8 +6,10 @@ # ./setup-providers.sh # register providers # # Usage: -# ./sandbox-podman.sh # uses agents/default.toml -# ./sandbox-podman.sh research # uses agents/research.toml +# ./sandbox-podman.sh # interactive, uses agents/default.toml +# ./sandbox-podman.sh research # uses agents/research.toml +# ./sandbox-podman.sh --name my-agent # named sandbox +# ./sandbox-podman.sh --no-tty --name test # non-interactive (for testing) # # To reconnect: openshell sandbox connect # To delete: openshell sandbox delete @@ -18,7 +20,19 @@ source "$SCRIPT_DIR/lib/common.sh" require_cli require_local_gateway -AGENT_NAME="${1:-default}" +# Parse args +AGENT_NAME="default" +SANDBOX_NAME="" +TTY_FLAG="--tty" + +while [[ $# -gt 0 ]]; do + case "$1" in + --name) SANDBOX_NAME="$2"; shift 2 ;; + --no-tty) TTY_FLAG="--no-tty"; shift ;; + *) AGENT_NAME="$1"; shift ;; + esac +done + AGENT_FILE="$SCRIPT_DIR/agents/${AGENT_NAME}.toml" [[ -f "$AGENT_FILE" ]] || { echo "ERROR: $AGENT_FILE not found."; exit 1; } @@ -48,17 +62,15 @@ for name in $SANDBOX_PROVIDERS; do fi done -# Stage files for upload to /sandbox/.config/openshell/ (same as OCP launcher) +# Stage files for upload to /sandbox/.config/openshell/ HARNESS_DIR="/tmp/openshell" mkdir -p "$HARNESS_DIR" -# Sandbox env vars from agent config if [[ -n "$SANDBOX_ENV" ]]; then echo "$SANDBOX_ENV" > "$HARNESS_DIR/sandbox.env" echo " Env: $(wc -l < "$HARNESS_DIR/sandbox.env") vars" fi -# GWS credentials if command -v gws &>/dev/null && gws auth status &>/dev/null; then gws auth export --unmasked > "$HARNESS_DIR/credentials.json" 2>/dev/null cp ~/.config/gws/client_secret.json "$HARNESS_DIR/client_secret.json" 2>/dev/null || true @@ -67,25 +79,38 @@ else echo " GWS: not configured (skipping)" fi -# Image flag +# Build flags FROM_FLAGS=() -if [[ -n "$SANDBOX_IMAGE" ]]; then - FROM_FLAGS=(--from "$SANDBOX_IMAGE") +[[ -n "$SANDBOX_IMAGE" ]] && FROM_FLAGS=(--from "$SANDBOX_IMAGE") + +NAME_FLAGS=() +[[ -n "$SANDBOX_NAME" ]] && NAME_FLAGS=(--name "$SANDBOX_NAME") + +# Command depends on tty mode +if [[ "$TTY_FLAG" == "--tty" ]]; then + CMD=(-- bash -c ". /sandbox/startup.sh && exec $SANDBOX_COMMAND") +else + CMD=(-- bash /sandbox/startup.sh) fi echo "" echo "=== Creating sandbox ===" -for attempt in 1 2 3; do +for attempt in 1 2 3 4 5; do "$CLI" sandbox create \ - --tty \ + $TTY_FLAG \ + ${NAME_FLAGS[@]+"${NAME_FLAGS[@]}"} \ ${FROM_FLAGS[@]+"${FROM_FLAGS[@]}"} \ ${PROVIDER_FLAGS[@]+"${PROVIDER_FLAGS[@]}"} \ --upload "$HARNESS_DIR:/sandbox/.config" --no-git-ignore \ - -- bash -c '. /sandbox/startup.sh && exec '"$SANDBOX_COMMAND" \ + "${CMD[@]}" \ && exit 0 echo " Attempt $attempt failed (supervisor race), retrying in 5s..." - "$CLI" sandbox delete "$("$CLI" sandbox list 2>/dev/null | awk 'NR==2{print $1}')" 2>/dev/null || true + if [[ -n "$SANDBOX_NAME" ]]; then + "$CLI" sandbox delete "$SANDBOX_NAME" 2>/dev/null || true + else + "$CLI" sandbox delete "$("$CLI" sandbox list 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' | awk 'NR==2{print $1}')" 2>/dev/null || true + fi sleep 5 done -echo "ERROR: Failed after 3 attempts." +echo "ERROR: Failed after 5 attempts." exit 1 diff --git a/test-flow.sh b/test-flow.sh index 93ac87c..1e0e2ee 100755 --- a/test-flow.sh +++ b/test-flow.sh @@ -122,63 +122,10 @@ test_podman() { check_providers if $FULL; then - # Parse agent config for image - local sandbox_image - sandbox_image=$(python3 -c " -import tomllib -with open('$SCRIPT_DIR/agents/default.toml', 'rb') as f: - print(tomllib.load(f).get('image', '')) -") - - local from_flags=() - [[ -n "$sandbox_image" ]] && from_flags=(--from "$sandbox_image") - - # Stage sandbox.env + GWS creds - local harness_dir="/tmp/test-harness/openshell" - rm -rf "$harness_dir" && mkdir -p "$harness_dir" - python3 -c " -import tomllib -with open('$SCRIPT_DIR/agents/default.toml', 'rb') as f: - env = tomllib.load(f).get('env', {}) -for k, v in env.items(): - print(f'export {k}={v}') -" > "$harness_dir/sandbox.env" - if command -v gws &>/dev/null && gws auth status &>/dev/null 2>&1; then - gws auth export --unmasked > "$harness_dir/credentials.json" 2>/dev/null - cp ~/.config/gws/client_secret.json "$harness_dir/client_secret.json" 2>/dev/null || true - fi - - # Create sandbox (non-interactive, retry on supervisor race) local sandbox_name="test-agent" - local created=false - local create_start=$(date +%s) - for attempt in 1 2 3 4 5; do - if "$CLI" sandbox create \ - --name "$sandbox_name" \ - --no-tty \ - ${from_flags[@]+"${from_flags[@]}"} \ - ${PROVIDER_FLAGS[@]+"${PROVIDER_FLAGS[@]}"} \ - --upload "$harness_dir:/sandbox/.config" --no-git-ignore \ - -- bash /sandbox/startup.sh \ - &>/dev/null; then - created=true - break - fi - "$CLI" sandbox delete "$sandbox_name" &>/dev/null || true - sleep 10 - done - - if $created; then - local create_elapsed=$(( $(date +%s) - create_start )) - printf " ✓ %-35s (%ds, %d attempts)\n" "sandbox create" "$create_elapsed" "$attempt" - ((PASS++)) - sandbox_verify "$sandbox_name" - else - printf " ✗ %-35s\n" "sandbox create (failed after 5 attempts)" - ((FAIL++)) - fi + step_output "sandbox create" "$SCRIPT_DIR/sandbox-podman.sh" --name "$sandbox_name" --no-tty + sandbox_verify "$sandbox_name" step "sandbox delete" "$CLI" sandbox delete "$sandbox_name" - rm -rf "$harness_dir" fi step "teardown (clean)" "$SCRIPT_DIR/teardown.sh" --sandboxes --providers