diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f122fda5d7..ec5c9112f0 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -73,6 +73,10 @@ Runtime-specific implementation notes belong in the driver crate README: - `crates/openshell-driver-kubernetes/README.md` - `crates/openshell-driver-vm/README.md` +The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun +executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves +that identity when it execs the supervisor after mounting and network setup. + ## Supervisor Delivery The supervisor must be available inside each sandbox workload: diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 2166e06177..d70e8c3ef5 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -9,6 +9,10 @@ set -euo pipefail +# libkrun consumes this driver-owned switch before exec'ing this script as +# PID 1. Do not leak the runtime control into the supervisor or workloads. +unset KRUN_INIT_PID1 + BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) # gvisor-tap-vsock subnet layout: # 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index a6ddfe011d..f6020af829 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -13,6 +13,7 @@ use std::time::{Duration, Instant}; use crate::{embedded_runtime, ffi, nft_ruleset, procguard, rootfs}; pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR"; +const KRUN_INIT_PID1_ENV: &str = "KRUN_INIT_PID1=1"; /// PID of the VM worker process (libkrun fork or QEMU). Zero when not running. /// Used by the SIGTERM/SIGINT handler to forward signals to the VM. @@ -833,15 +834,7 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { vm.set_console_output(&config.console_output)?; - let env = if config.env.is_empty() { - vec![ - "HOME=/root".to_string(), - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), - "TERM=xterm".to_string(), - ] - } else { - config.env.clone() - }; + let env = libkrun_guest_env(config); vm.set_exec(&config.exec_path, &config.args, &env)?; let pid = unsafe { libc::fork() }; @@ -889,6 +882,26 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { } } +fn libkrun_guest_env(config: &VmLaunchConfig) -> Vec { + let mut env = if config.env.is_empty() { + vec![ + "HOME=/root".to_string(), + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), + "TERM=xterm".to_string(), + ] + } else { + config.env.clone() + }; + + // libkrun normally keeps /init.krun as PID 1 and forks the configured + // executable. OpenShell's guest init is itself an init process and ends + // by exec'ing the supervisor, so ask libkrun to exec it directly. Keep + // this driver-owned setting authoritative over sandbox image/user env. + env.retain(|value| !value.starts_with("KRUN_INIT_PID1=")); + env.push(KRUN_INIT_PID1_ENV.to_string()); + env +} + pub fn validate_runtime_dir(dir: &Path) -> Result<(), String> { if !dir.is_dir() { return Err(format!( @@ -1438,6 +1451,44 @@ mod tests { assert!(env.contains(&"GPU_ENABLED=true".to_string())); } + #[test] + fn libkrun_guest_env_runs_guest_init_as_pid_one() { + let env = libkrun_guest_env(&qemu_config()); + + assert!(env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string())); + assert!(env.contains(&KRUN_INIT_PID1_ENV.to_string())); + } + + #[test] + fn libkrun_guest_env_overrides_caller_pid_one_setting() { + let mut config = qemu_config(); + config.env.extend([ + "KRUN_INIT_PID1=0".to_string(), + "KRUN_INIT_PID1=unexpected".to_string(), + ]); + + let env = libkrun_guest_env(&config); + let pid_one_settings = env + .iter() + .filter(|value| value.starts_with("KRUN_INIT_PID1=")) + .collect::>(); + + assert_eq!(pid_one_settings.len(), 1); + assert_eq!(pid_one_settings[0], KRUN_INIT_PID1_ENV); + } + + #[test] + fn libkrun_guest_env_keeps_defaults_when_no_env_is_configured() { + let mut config = qemu_config(); + config.env.clear(); + + let env = libkrun_guest_env(&config); + + assert!(env.contains(&"HOME=/root".to_string())); + assert!(env.contains(&"TERM=xterm".to_string())); + assert!(env.contains(&KRUN_INIT_PID1_ENV.to_string())); + } + #[test] fn kernel_cmdline_keeps_guest_init_metadata_out_of_proc_cmdline() { let cmdline = build_kernel_cmdline(&qemu_config()); diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index c6fec9aeb8..09cb02a702 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -18,8 +18,8 @@ publish = false [features] e2e = [] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable -# alias to the host running test fixtures). docker and podman wire the alias -# unconditionally; the kube driver only does so when the chart's +# alias to the host running test fixtures). docker, podman, and vm wire the +# alias unconditionally; the kube driver only does so when the chart's # `server.hostGatewayIP` is set, so `e2e-kubernetes` does NOT imply this and # the helm wrapper opts in explicitly when it has resolved an IP. e2e-host-gateway = ["e2e"] @@ -30,7 +30,7 @@ e2e-docker-gpu = ["e2e-docker", "e2e-gpu"] e2e-kubernetes = ["e2e"] e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] -e2e-vm = ["e2e"] +e2e-vm = ["e2e", "e2e-host-gateway"] [[test]] name = "custom_image" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 467d419f02..ef798a8ea9 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -31,7 +31,7 @@ # 4. Writes a per-run gateway config with `[openshell.drivers.vm]` # settings, starts the gateway with `--config /gateway.toml` # on a random free port, waits for `Server listening`, then runs the -# selected Rust e2e test (`smoke` by default). +# selected Rust e2e tests. # 5. Tears the gateway down and (on failure) preserves the gateway # log and every VM serial console log for post-mortem. # @@ -47,7 +47,7 @@ source "${ROOT}/e2e/support/gateway-common.sh" COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm" -E2E_TEST="${OPENSHELL_E2E_VM_TEST:-smoke}" +E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. @@ -296,11 +296,23 @@ e2e_export_gateway_restart_metadata \ # preparation; allow 180s for slower CI runners. export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -echo "==> Running e2e ${E2E_TEST} test (features: ${E2E_FEATURES}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})" -cargo test \ - --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ - --features "${E2E_FEATURES}" \ - --test "${E2E_TEST}" \ - -- --nocapture +run_e2e_test() { + local test_target="$1" + shift + + echo "==> Running e2e ${test_target} test (features: ${E2E_FEATURES}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})" + cargo test \ + --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ + --features "${E2E_FEATURES}" \ + --test "${test_target}" \ + "$@" \ + -- --nocapture + echo "==> ${test_target} test passed." +} -echo "==> ${E2E_TEST} test passed." +if [ -n "${E2E_TEST_OVERRIDE}" ]; then + run_e2e_test "${E2E_TEST_OVERRIDE}" +else + run_e2e_test smoke + run_e2e_test host_gateway_alias +fi diff --git a/tasks/test.toml b/tasks/test.toml index db3878f756..4e26799758 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -120,7 +120,7 @@ env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } run = "e2e/rust/e2e-kubernetes.sh" ["e2e:vm"] -description = "Start openshell-gateway with the VM compute driver and run the cluster-agnostic smoke e2e" +description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh" ["e2e:docker"]