From de381ea840773f24afc87e5236b5748e7bbac50b Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 16 Jul 2026 15:01:54 +0200 Subject: [PATCH 1/7] test(e2e): add API conformance suite Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 35 + e2e/rust/Cargo.lock | 1005 ++++++++++++++++++++- e2e/rust/Cargo.toml | 18 +- e2e/rust/e2e-kubernetes.sh | 2 +- e2e/rust/e2e-podman.sh | 2 +- e2e/rust/e2e-vm.sh | 9 +- e2e/rust/src/bin/openshell-conformance.rs | 157 ++++ e2e/rust/src/conformance.rs | 771 ++++++++++++++++ e2e/rust/src/lib.rs | 1 + e2e/rust/tests/conformance.rs | 44 + e2e/with-docker-gateway.sh | 4 + e2e/with-kube-gateway.sh | 2 + e2e/with-podman-gateway.sh | 4 + tasks/test.toml | 4 +- 14 files changed, 2038 insertions(+), 20 deletions(-) create mode 100644 e2e/rust/CONFORMANCE.md create mode 100644 e2e/rust/src/bin/openshell-conformance.rs create mode 100644 e2e/rust/src/conformance.rs create mode 100644 e2e/rust/tests/conformance.rs diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md new file mode 100644 index 0000000000..aa885975cd --- /dev/null +++ b/e2e/rust/CONFORMANCE.md @@ -0,0 +1,35 @@ +# OpenShell conformance + +`openshell-conformance` is an internal CLI for validating the driver-agnostic +behavior of an existing OpenShell gateway installation. Its scenario engine +lives in the standalone `e2e/rust` package and is shared with the Rust +`e2e-api-conformance` test profile. + +The binary is not published or included in release packaging. Build and run it +through the e2e manifest: + +```shell +cargo run --manifest-path e2e/rust/Cargo.toml --bin openshell-conformance -- list +cargo run --manifest-path e2e/rust/Cargo.toml --bin openshell-conformance -- \ + run --gateway-endpoint http://127.0.0.1:50051 +cargo run --manifest-path e2e/rust/Cargo.toml --bin openshell-conformance -- \ + run --gateway-endpoint http://127.0.0.1:50051 --filter lifecycle --timeout 120 +``` + +The runner requires an explicit gateway endpoint, either through +`--gateway-endpoint` or `OPENSHELL_GATEWAY_ENDPOINT`. HTTPS gateways require +explicit `--tls-ca`, `--tls-cert`, and `--tls-key` paths. Each created sandbox +uses the name +`conformance--`. The runner performs best-effort cleanup, and +the prefix makes any sandbox left after an interruption easy to identify. + +The capabilities scenario remains deferred until the gateway exposes driver +capabilities through its public API. It is not included as a known-failing CI +scenario. + +The `e2e-api-conformance` profile invokes the scenario engine directly against +every gateway driver. + +Gateway provisioners export the endpoint and any mTLS paths through +`OPENSHELL_GATEWAY_ENDPOINT` and the `OPENSHELL_CONFORMANCE_TLS_*` variables. +Set `OPENSHELL_CONFORMANCE_TIMEOUT` to change the per-scenario timeout. diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index aebec66c09..ab3996e86d 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2,18 +2,179 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.22.1" @@ -84,12 +245,84 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "clap" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -158,6 +391,24 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -245,6 +496,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -270,6 +532,37 @@ dependencies = [ "wasip3", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -352,6 +645,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -378,6 +672,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -534,6 +841,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_ci" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itertools" version = "0.14.0" @@ -598,12 +923,63 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "miette" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f98efec8807c63c752b5bd61f862c165c115b0a35685bdcfd9238c7aeb592b7" +dependencies = [ + "backtrace", + "backtrace-ext", + "cfg-if", + "miette-derive", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", + "unicode-width 0.1.14", +] + +[[package]] +name = "miette-derive" +version = "7.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.1.1" @@ -615,12 +991,55 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openshell-core" +version = "0.0.0" +dependencies = [ + "base64", + "glob", + "ipnet", + "miette", + "prost 0.14.4", + "prost-types", + "protobuf-src", + "serde", + "serde_json", + "thiserror", + "tokio", + "tonic", + "tonic-prost", + "tonic-prost-build", + "tracing", + "url", +] + [[package]] name = "openshell-e2e" version = "0.1.0" @@ -628,12 +1047,15 @@ dependencies = [ "base64", "bollard", "bytes", + "clap", "futures-util", "hex", "http-body-util", "hyper", "hyper-util", - "prost", + "miette", + "openshell-core", + "prost 0.13.5", "rand", "serde", "serde_json", @@ -643,8 +1065,21 @@ dependencies = [ "sha2", "tempfile", "tokio", + "tonic", ] +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "owo-colors" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" + [[package]] name = "parking_lot" version = "0.12.5" @@ -674,6 +1109,37 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -724,7 +1190,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive 0.14.4", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost 0.14.4", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn", + "tempfile", ] [[package]] @@ -740,6 +1237,57 @@ dependencies = [ "syn", ] +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost 0.14.4", +] + +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +dependencies = [ + "bitflags", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] + [[package]] name = "quote" version = "1.0.45" @@ -799,6 +1347,55 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustix" version = "1.1.4" @@ -812,18 +1409,96 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" +dependencies = [ + "log", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "ryu" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -958,6 +1633,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -996,6 +1677,39 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "supports-color" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fc7232dd8d2e4ac5ce4ef302b1d81e0b80d055b9d77c7c4f51f6aa4c867d6" +dependencies = [ + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e396b6523b11ccb83120b115a0b7366de372751aa6edf19844dfb13a6af97e91" + +[[package]] +name = "supports-unicode" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" + [[package]] name = "syn" version = "2.0.117" @@ -1007,6 +1721,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + [[package]] name = "synstructure" version = "0.13.2" @@ -1031,6 +1751,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "unicode-linebreak", + "unicode-width 0.2.2", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -1089,6 +1829,27 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -1102,6 +1863,101 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" +dependencies = [ + "async-trait", + "axum", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "rustls-native-certs", + "socket2", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f61875ac5293cf72e6c8cf0158086428c82c37229e98c840878f1706b0322" +dependencies = [ + "prettyplease", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tonic-prost" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" +dependencies = [ + "bytes", + "prost 0.14.4", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "654e5643eff75d7f8c99197ce1440ed19a3474eada74c12bbac488b2cafdae27" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -1115,9 +1971,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", + "tracing-attributes", "tracing-core", ] +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.36" @@ -1139,18 +2007,48 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1169,6 +2067,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "version_check" version = "0.9.5" @@ -1270,13 +2174,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets", + "windows-targets 0.53.5", ] [[package]] @@ -1288,6 +2201,22 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + [[package]] name = "windows-targets" version = "0.53.5" @@ -1295,58 +2224,106 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ "windows-link", - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + [[package]] name = "windows_aarch64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + [[package]] name = "windows_aarch64_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + [[package]] name = "windows_i686_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + [[package]] name = "windows_i686_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + [[package]] name = "windows_i686_msvc" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + [[package]] name = "windows_x86_64_gnu" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + [[package]] name = "windows_x86_64_gnullvm" version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "windows_x86_64_msvc" version = "0.53.1" @@ -1511,6 +2488,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + [[package]] name = "zerotrie" version = "0.2.4" diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 09cb02a702..d94de47899 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Standalone crate — the empty [workspace] table prevents Cargo from -# treating this as part of the root workspace, avoiding Dockerfile -# and Cargo.lock coupling. +# Standalone crate — the empty [workspace] table keeps e2e targets and its +# lockfile out of the root workspace. The internal conformance runner +# intentionally reuses openshell-core protocol types through a path dependency. [workspace] [package] @@ -11,12 +11,13 @@ name = "openshell-e2e" description = "End-to-end tests for the OpenShell CLI" version = "0.1.0" edition = "2024" -rust-version = "1.88" +rust-version = "1.90" license = "Apache-2.0" publish = false [features] e2e = [] +e2e-api-conformance = ["e2e"] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the # alias unconditionally; the kube driver only does so when the chart's @@ -32,6 +33,11 @@ e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"] e2e-podman-gpu = ["e2e-podman", "e2e-gpu"] e2e-vm = ["e2e", "e2e-host-gateway"] +[[test]] +name = "conformance" +path = "tests/conformance.rs" +required-features = ["e2e-api-conformance"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" @@ -111,13 +117,17 @@ required-features = ["e2e-gpu"] base64 = "0.22" bollard = "0.20" bytes = "1" +clap = { version = "4.5", features = ["derive", "env"] } futures-util = "0.3" http-body-util = "0.1" hyper = { version = "1", features = ["client", "http1"] } hyper-util = { version = "0.1", features = ["tokio"] } +miette = "7" +openshell-core = { path = "../../crates/openshell-core", default-features = false } prost = "0.13" tokio = { version = "1.43", features = ["full"] } tempfile = "3" +tonic = { version = "0.14", features = ["tls-native-roots"] } sha1 = "0.10" sha2 = "0.10" hex = "0.4" diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index ec0faefed5..7b5ab8bbc1 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -19,7 +19,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-host-gateway,e2e-kubernetes}" +E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-api-conformance,e2e-host-gateway,e2e-kubernetes}" cargo build -p openshell-cli diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 39b6b523a8..d698db55e5 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}" +E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman,e2e-api-conformance}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" cargo build -p openshell-cli diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 8005d09bd0..bfb85cd073 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -48,7 +48,7 @@ COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" +E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm,e2e-api-conformance}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. # AF_UNIX SUN_LEN is 104 bytes on macOS (108 on Linux), so paths anchored @@ -281,6 +281,9 @@ e2e_register_mtls_gateway \ "${PKI_DIR}" export OPENSHELL_GATEWAY_ENDPOINT="${CLI_GATEWAY_ENDPOINT}" +export OPENSHELL_CONFORMANCE_TLS_CA="${PKI_DIR}/ca.crt" +export OPENSHELL_CONFORMANCE_TLS_CERT="${PKI_DIR}/client/tls.crt" +export OPENSHELL_CONFORMANCE_TLS_KEY="${PKI_DIR}/client/tls.key" export OPENSHELL_E2E_EXPECT_VM_OVERLAY=1 export OPENSHELL_E2E_DRIVER="vm" export OPENSHELL_E2E_VM_STATE_DIR="${RUN_STATE_DIR}" @@ -317,3 +320,7 @@ else run_e2e_test host_gateway_alias run_e2e_test vm_gateway_resume fi + +if [ "${E2E_TEST_OVERRIDE}" != "conformance" ]; then + run_e2e_test conformance +fi diff --git a/e2e/rust/src/bin/openshell-conformance.rs b/e2e/rust/src/bin/openshell-conformance.rs new file mode 100644 index 0000000000..1a9c20cc56 --- /dev/null +++ b/e2e/rust/src/bin/openshell-conformance.rs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Internal driver-agnostic conformance runner for `OpenShell` installations. + +use std::path::PathBuf; + +use clap::{Parser, Subcommand, ValueEnum}; +use miette::{Result, WrapErr}; +use openshell_e2e::conformance::ConnectionOptions; + +#[derive(Parser, Debug)] +#[command( + name = "openshell-conformance", + about = "Validate an OpenShell gateway and its configured compute driver" +)] +struct Cli { + /// Gateway endpoint to test. + #[arg(long, global = true, env = "OPENSHELL_GATEWAY_ENDPOINT")] + gateway_endpoint: Option, + + /// Path to the gateway CA certificate. + #[arg(long, global = true)] + tls_ca: Option, + + /// Path to the client certificate. + #[arg(long, global = true)] + tls_cert: Option, + + /// Path to the client private key. + #[arg(long, global = true)] + tls_key: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand, Debug)] +enum Command { + /// Run conformance scenarios against a gateway. + Run { + /// Only run scenarios whose name contains this substring. + #[arg(long, short = 'f')] + filter: Option, + + /// Maximum duration of each scenario, in seconds. + #[arg(long, default_value_t = 300)] + timeout: u64, + + /// Output format. + #[arg(short = 'o', long, value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, + + /// List available conformance scenarios without connecting to a gateway. + List { + /// Output format. + #[arg(short = 'o', long, value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +enum OutputFormat { + Table, + Yaml, + Json, +} + +impl OutputFormat { + const fn as_str(self) -> &'static str { + match self { + Self::Table => "table", + Self::Yaml => "yaml", + Self::Json => "json", + } + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.command { + Command::List { output } => openshell_e2e::conformance::conformance_list(output.as_str()), + Command::Run { + filter, + timeout, + output, + } => { + let gateway_endpoint = cli.gateway_endpoint.ok_or_else(|| { + miette::miette!( + "gateway endpoint is required; pass --gateway-endpoint or set \ + OPENSHELL_GATEWAY_ENDPOINT" + ) + })?; + let connection = ConnectionOptions { + tls_ca: cli.tls_ca, + tls_cert: cli.tls_cert, + tls_key: cli.tls_key, + }; + + openshell_e2e::conformance::conformance_run( + &gateway_endpoint, + &connection, + filter.as_deref(), + timeout, + output.as_str(), + ) + .await + .wrap_err_with(|| format!("conformance failed for {gateway_endpoint}")) + } + } +} + +#[cfg(test)] +mod tests { + use super::{Cli, Command, OutputFormat}; + use clap::Parser; + + #[test] + fn list_requires_no_gateway_arguments() { + let cli = Cli::try_parse_from(["openshell-conformance", "list"]).unwrap(); + assert!(matches!( + cli.command, + Command::List { + output: OutputFormat::Table + } + )); + } + + #[test] + fn run_accepts_filter_timeout_and_structured_output() { + let cli = Cli::try_parse_from([ + "openshell-conformance", + "run", + "--gateway-endpoint", + "http://127.0.0.1:50051", + "--filter", + "lifecycle", + "--timeout", + "30", + "--output", + "json", + ]) + .unwrap(); + + assert!(matches!( + cli.command, + Command::Run { + filter: Some(ref filter), + timeout: 30, + output: OutputFormat::Json, + } if filter == "lifecycle" + )); + } +} diff --git a/e2e/rust/src/conformance.rs b/e2e/rust/src/conformance.rs new file mode 100644 index 0000000000..4172361e63 --- /dev/null +++ b/e2e/rust/src/conformance.rs @@ -0,0 +1,771 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Driver-agnostic conformance scenarios for `OpenShell` gateway installations. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; + +use futures_util::StreamExt; +use miette::{IntoDiagnostic, Result, WrapErr}; +use openshell_core::proto::{ + CreateSandboxRequest, DeleteSandboxRequest, GetSandboxRequest, GpuResourceRequirements, + ListSandboxesRequest, ResourceRequirements, SandboxPhase, SandboxSpec, WatchSandboxRequest, + open_shell_client::OpenShellClient, +}; +use openshell_core::{ObjectId, ObjectName}; +use tonic::Code; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity}; + +type GrpcClient = OpenShellClient; + +/// Explicit connection material for an existing gateway installation. +#[derive(Clone, Debug, Default)] +pub struct ConnectionOptions { + /// Path to the gateway CA certificate. + pub tls_ca: Option, + /// Path to the client certificate. + pub tls_cert: Option, + /// Path to the client private key. + pub tls_key: Option, +} + +async fn grpc_client(server: &str, options: &ConnectionOptions) -> Result { + let mut endpoint = Endpoint::from_shared(server.to_string()) + .into_diagnostic()? + .connect_timeout(Duration::from_secs(10)) + .http2_adaptive_window(true) + .http2_keep_alive_interval(Duration::from_secs(10)) + .keep_alive_while_idle(true); + + if server.starts_with("https://") { + let ca_path = options + .tls_ca + .as_ref() + .ok_or_else(|| miette::miette!("--tls-ca is required for HTTPS gateways"))?; + let cert_path = options + .tls_cert + .as_ref() + .ok_or_else(|| miette::miette!("--tls-cert is required for HTTPS gateways"))?; + let key_path = options + .tls_key + .as_ref() + .ok_or_else(|| miette::miette!("--tls-key is required for HTTPS gateways"))?; + let ca = std::fs::read(ca_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read TLS CA from {}", ca_path.display()))?; + let cert = std::fs::read(cert_path) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to read TLS certificate from {}", + cert_path.display() + ) + })?; + let key = std::fs::read(key_path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read TLS key from {}", key_path.display()))?; + endpoint = endpoint + .tls_config( + ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(ca)) + .identity(Identity::from_pem(cert, key)), + ) + .into_diagnostic()?; + } + + let channel = endpoint + .connect() + .await + .into_diagnostic() + .wrap_err("failed to connect to gateway")?; + Ok(OpenShellClient::new(channel)) +} + +// ----------------------------------------------------------------------- +// Conformance suite +// ----------------------------------------------------------------------- + +/// A single conformance scenario definition. +struct Scenario { + /// Short lowercase-hyphenated name, used in sandbox naming and filtering. + name: &'static str, + /// Human-readable description shown in `conformance list`. + description: &'static str, +} + +fn all_scenarios() -> &'static [Scenario] { + &[ + Scenario { + name: "lifecycle", + description: "Create → running → stop → delete completes without error", + }, + Scenario { + name: "not-found", + description: "Get/stop/delete for an unknown sandbox ID returns an appropriate error", + }, + Scenario { + name: "idempotent-delete", + description: "Deleting an already-deleted sandbox does not error", + }, + Scenario { + name: "validate", + description: "Invalid sandbox specs are rejected before creation", + }, + Scenario { + name: "concurrent", + description: "Two sandboxes created simultaneously do not interfere", + }, + Scenario { + name: "labels", + description: "Labels are persisted on create and filter list results correctly", + }, + ] +} + +/// Outcome of a single conformance scenario. +#[derive(serde::Serialize)] +struct ScenarioResult { + name: String, + passed: bool, + message: String, + duration_ms: u64, +} + +pub async fn conformance_run( + server: &str, + connection: &ConnectionOptions, + filter: Option<&str>, + timeout_secs: u64, + output: &str, +) -> Result<()> { + use std::time::Instant; + + let client = grpc_client(server, connection).await?; + + // Stable run-id for sandbox naming: seconds since Unix epoch, truncated + // to 8 hex digits. Keeps names Kubernetes RFC 1123 safe and short enough + // to read in `sandbox list` output. + let run_id = format!( + "{:08x}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + & u64::from(u32::MAX) + ); + + let scenarios = all_scenarios() + .iter() + .filter(|s| filter.is_none_or(|f| s.name.contains(f))); + + let mut results: Vec = Vec::new(); + let mut any_failed = false; + + for scenario in scenarios { + let start = Instant::now(); + let timeout = Duration::from_secs(timeout_secs); + let mut client = client.clone(); + + let outcome = + tokio::time::timeout(timeout, run_scenario(scenario.name, &mut client, &run_id)) + .await + .unwrap_or_else(|_| { + Err(miette::miette!("scenario timed out after {timeout_secs}s")) + }); + + let passed = outcome.is_ok(); + if !passed { + any_failed = true; + } + + results.push(ScenarioResult { + name: scenario.name.to_string(), + passed, + message: match &outcome { + Ok(()) => "ok".to_string(), + Err(e) => format!("{e}"), + }, + duration_ms: u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX), + }); + } + + match output { + "json" => println!( + "{}", + serde_json::to_string_pretty(&results).into_diagnostic()? + ), + "yaml" => print!("{}", serde_yml::to_string(&results).into_diagnostic()?), + "table" => { + for result in &results { + let status = if result.passed { "PASS" } else { "FAIL" }; + println!( + " [{status}] {} ({}ms) — {}", + result.name, result.duration_ms, result.message + ); + } + } + _ => return Err(miette::miette!("unsupported output format: {output}")), + } + + if any_failed { + return Err(miette::miette!("one or more conformance scenarios failed")); + } + Ok(()) +} + +/// Dispatch a scenario by name. +async fn run_scenario(name: &str, client: &mut GrpcClient, run_id: &str) -> Result<()> { + match name { + "lifecycle" => scenario_lifecycle(client, run_id).await, + "not-found" => scenario_not_found(client, run_id).await, + "idempotent-delete" => scenario_idempotent_delete(client, run_id).await, + "validate" => scenario_validate(client).await, + "concurrent" => scenario_concurrent(client, run_id).await, + "labels" => scenario_labels(client, run_id).await, + _ => Err(miette::miette!("scenario '{name}' is not yet implemented")), + } +} + +/// Poll `WatchSandbox` until the sandbox reaches Ready, returning an error on +/// Error phase or a closed stream. Does not perform cleanup — callers are +/// responsible for deleting the sandbox if this returns an error. +async fn wait_for_ready( + client: &mut GrpcClient, + sandbox_id: &str, + sandbox_name: &str, +) -> Result<()> { + let mut stream = client + .watch_sandbox(WatchSandboxRequest { + id: sandbox_id.to_string(), + follow_status: true, + follow_logs: false, + follow_events: false, + log_tail_lines: 0, + event_tail: 0, + stop_on_terminal: false, + log_since_ms: 0, + log_sources: vec![], + log_min_level: String::new(), + }) + .await + .into_diagnostic() + .wrap_err("watch_sandbox failed")? + .into_inner(); + + while let Some(item) = stream.next().await { + let evt = item + .into_diagnostic() + .wrap_err("watch_sandbox stream error")?; + if let Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(s)) = evt.payload + { + match SandboxPhase::try_from(s.phase()).unwrap_or(SandboxPhase::Unknown) { + SandboxPhase::Ready => return Ok(()), + SandboxPhase::Error => { + return Err(miette::miette!( + "sandbox '{sandbox_name}' entered Error phase before becoming Ready" + )); + } + _ => {} + } + } + } + + Err(miette::miette!( + "watch stream ended before sandbox '{sandbox_name}' reached Ready" + )) +} + +/// Scenario: create → ready → delete. +/// +/// Creates a minimal sandbox, waits for it to reach the Ready phase, then +/// deletes it. Verifies that the sandbox appears in the list between create +/// and delete, and that delete reports it as deleted. +async fn scenario_lifecycle(client: &mut GrpcClient, run_id: &str) -> Result<()> { + let sandbox_name = format!("conformance-lifecycle-{run_id}"); + + // ── 1. Create ──────────────────────────────────────────────────────── + let response = client + .create_sandbox(CreateSandboxRequest { + name: sandbox_name.clone(), + spec: Some(SandboxSpec::default()), + labels: HashMap::default(), + annotations: HashMap::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox failed")?; + + let sandbox = response + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; + let sandbox_id = sandbox.object_id().to_string(); + + // ── 2. Wait for Ready ──────────────────────────────────────────────── + if let Err(e) = wait_for_ready(client, &sandbox_id, &sandbox_name).await { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(e); + } + + // ── 3. Verify it appears in the list ──────────────────────────────── + let list_response = client + .list_sandboxes(ListSandboxesRequest::default()) + .await + .into_diagnostic() + .wrap_err("list_sandboxes failed")?; + + let found = list_response + .into_inner() + .sandboxes + .iter() + .any(|s| s.object_name() == sandbox_name); + + if !found { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(miette::miette!( + "sandbox '{sandbox_name}' not found in list_sandboxes response after creation" + )); + } + + // ── 4. Delete ──────────────────────────────────────────────────────── + let del_response = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox failed")?; + + if !del_response.into_inner().deleted { + return Err(miette::miette!( + "delete_sandbox reported sandbox '{sandbox_name}' was not deleted" + )); + } + + Ok(()) +} + +/// Scenario: get and delete a sandbox that does not exist. +/// +/// Verifies that `GetSandbox` returns `NOT_FOUND` and that `DeleteSandbox` +/// returns `deleted: false` without erroring for a name that was never +/// created. +async fn scenario_not_found(client: &mut GrpcClient, run_id: &str) -> Result<()> { + let phantom_name = format!("conformance-not-found-{run_id}"); + + // ── 1. GetSandbox → NOT_FOUND ──────────────────────────────────────── + let err = client + .get_sandbox(GetSandboxRequest { + name: phantom_name.clone(), + }) + .await + .expect_err("get_sandbox on a non-existent sandbox should have returned NOT_FOUND"); + + if err.code() != Code::NotFound { + return Err(miette::miette!( + "get_sandbox returned {} instead of NOT_FOUND", + err.code() + )); + } + + // ── 2. DeleteSandbox → ok, deleted: false ──────────────────────────── + let del = client + .delete_sandbox(DeleteSandboxRequest { + name: phantom_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox on a non-existent sandbox should not error")? + .into_inner(); + + if del.deleted { + return Err(miette::miette!( + "delete_sandbox reported deleted=true for a sandbox that was never created" + )); + } + + Ok(()) +} + +/// Scenario: delete an already-deleted sandbox does not error. +/// +/// Creates a sandbox, waits for it to be Ready, deletes it (expecting +/// `deleted: true`), then deletes it again and asserts the second call +/// returns `deleted: false` without an error. +async fn scenario_idempotent_delete(client: &mut GrpcClient, run_id: &str) -> Result<()> { + let sandbox_name = format!("conformance-idempotent-delete-{run_id}"); + + // ── 1. Create ──────────────────────────────────────────────────────── + let response = client + .create_sandbox(CreateSandboxRequest { + name: sandbox_name.clone(), + spec: Some(SandboxSpec::default()), + labels: HashMap::default(), + annotations: HashMap::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox failed")?; + + let sandbox = response + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; + let sandbox_id = sandbox.object_id().to_string(); + + // ── 2. Wait for Ready ──────────────────────────────────────────────── + if let Err(e) = wait_for_ready(client, &sandbox_id, &sandbox_name).await { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await; + return Err(e); + } + + // ── 3. First delete → deleted: true ────────────────────────────────── + let del1 = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("first delete_sandbox failed")? + .into_inner(); + + if !del1.deleted { + return Err(miette::miette!( + "first delete_sandbox reported deleted=false for sandbox '{sandbox_name}'" + )); + } + + // ── 4. Second delete → ok, deleted: false ──────────────────────────── + let del2 = client + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.clone(), + }) + .await + .into_diagnostic() + .wrap_err("second delete_sandbox (idempotency check) returned an error")? + .into_inner(); + + if del2.deleted { + return Err(miette::miette!( + "second delete_sandbox reported deleted=true — expected deleted=false" + )); + } + + Ok(()) +} + +/// Scenario: invalid sandbox specs are rejected before creation. +/// +/// Verifies that the gateway returns `INVALID_ARGUMENT` for two distinct +/// invalid inputs without creating any sandbox: a missing spec field and a +/// zero GPU count. +async fn scenario_validate(client: &mut GrpcClient) -> Result<()> { + // ── 1. spec=None → INVALID_ARGUMENT ────────────────────────────────── + let err = client + .create_sandbox(CreateSandboxRequest { + name: String::new(), + spec: None, + labels: HashMap::default(), + annotations: HashMap::default(), + }) + .await + .expect_err("create_sandbox with spec=None should have been rejected"); + + if err.code() != Code::InvalidArgument { + return Err(miette::miette!( + "create_sandbox(spec=None) returned {} instead of INVALID_ARGUMENT", + err.code() + )); + } + + // ── 2. gpu.count=0 → INVALID_ARGUMENT ──────────────────────────────── + let err2 = client + .create_sandbox(CreateSandboxRequest { + name: String::new(), + spec: Some(SandboxSpec { + resource_requirements: Some(ResourceRequirements { + gpu: Some(GpuResourceRequirements { count: Some(0) }), + }), + ..Default::default() + }), + labels: HashMap::default(), + annotations: HashMap::default(), + }) + .await + .expect_err("create_sandbox with gpu.count=0 should have been rejected"); + + if err2.code() != Code::InvalidArgument { + return Err(miette::miette!( + "create_sandbox(gpu.count=0) returned {} instead of INVALID_ARGUMENT", + err2.code() + )); + } + + Ok(()) +} + +/// Scenario: two sandboxes created simultaneously do not interfere. +/// +/// Issues two `CreateSandbox` calls concurrently, waits for both to reach +/// Ready in parallel, verifies both appear in `ListSandboxes`, then deletes +/// both. Any failure cleans up both sandboxes before returning. +async fn scenario_concurrent(client: &mut GrpcClient, run_id: &str) -> Result<()> { + let name_a = format!("conformance-concurrent-a-{run_id}"); + let name_b = format!("conformance-concurrent-b-{run_id}"); + + // ── 1. Create both sandboxes concurrently ──────────────────────────── + let mut client_a = client.clone(); + let mut client_b = client.clone(); + + let (resp_a, resp_b) = tokio::join!( + client_a.create_sandbox(CreateSandboxRequest { + name: name_a.clone(), + spec: Some(SandboxSpec::default()), + labels: HashMap::default(), + annotations: HashMap::default(), + }), + client_b.create_sandbox(CreateSandboxRequest { + name: name_b.clone(), + spec: Some(SandboxSpec::default()), + labels: HashMap::default(), + annotations: HashMap::default(), + }), + ); + + let sandbox_a = resp_a + .into_diagnostic() + .wrap_err("create_sandbox(a) failed")? + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox(a) response missing sandbox"))?; + let sandbox_b = resp_b + .into_diagnostic() + .wrap_err("create_sandbox(b) failed")? + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox(b) response missing sandbox"))?; + + let id_a = sandbox_a.object_id().to_string(); + let id_b = sandbox_b.object_id().to_string(); + + // ── 2. Wait for both to reach Ready concurrently ───────────────────── + let (ready_a, ready_b) = tokio::join!( + wait_for_ready(&mut client_a, &id_a, &name_a), + wait_for_ready(&mut client_b, &id_b, &name_b), + ); + + // Best-effort cleanup before surfacing watch errors. + if ready_a.is_err() || ready_b.is_err() { + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_a.clone(), + }) + .await; + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_b.clone(), + }) + .await; + ready_a?; + ready_b?; + } + + // ── 3. Both appear in the list ─────────────────────────────────────── + let sandboxes = client + .list_sandboxes(ListSandboxesRequest::default()) + .await + .into_diagnostic() + .wrap_err("list_sandboxes failed")? + .into_inner() + .sandboxes; + + let found_a = sandboxes.iter().any(|s| s.object_name() == name_a); + let found_b = sandboxes.iter().any(|s| s.object_name() == name_b); + + // ── 4. Delete both ─────────────────────────────────────────────────── + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_a.clone(), + }) + .await; + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_b.clone(), + }) + .await; + + if !found_a { + return Err(miette::miette!( + "sandbox '{name_a}' not found in list_sandboxes after concurrent create" + )); + } + if !found_b { + return Err(miette::miette!( + "sandbox '{name_b}' not found in list_sandboxes after concurrent create" + )); + } + + Ok(()) +} + +/// Scenario: labels are persisted on create and filter list results correctly. +/// +/// Creates two sandboxes with distinct label values, calls `ListSandboxes` +/// with a label selector, and verifies that only the matching sandbox is +/// returned. Labels are stored by the gateway on creation so no Ready wait +/// is required; both sandboxes are deleted after the assertion. +async fn scenario_labels(client: &mut GrpcClient, run_id: &str) -> Result<()> { + let name_a = format!("conformance-labels-a-{run_id}"); + let name_b = format!("conformance-labels-b-{run_id}"); + let label_key = "conformance-scenario".to_string(); + + // ── 1. Create two sandboxes with distinct label values ─────────────── + client + .create_sandbox(CreateSandboxRequest { + name: name_a.clone(), + spec: Some(SandboxSpec::default()), + labels: [(label_key.clone(), "labels-a".to_string())] + .into_iter() + .collect(), + annotations: HashMap::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox(a) failed")?; + + client + .create_sandbox(CreateSandboxRequest { + name: name_b.clone(), + spec: Some(SandboxSpec::default()), + labels: [(label_key.clone(), "labels-b".to_string())] + .into_iter() + .collect(), + annotations: HashMap::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox(b) failed")?; + + // ── 2. Filter by label — must return only sandbox A ────────────────── + let filtered = client + .list_sandboxes(ListSandboxesRequest { + label_selector: format!("{label_key}=labels-a"), + ..Default::default() + }) + .await + .into_diagnostic() + .wrap_err("list_sandboxes with label_selector failed")? + .into_inner() + .sandboxes; + + // ── 3. Cleanup ─────────────────────────────────────────────────────── + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_a.clone(), + }) + .await; + let _ = client + .delete_sandbox(DeleteSandboxRequest { + name: name_b.clone(), + }) + .await; + + // ── 4. Assert after cleanup so both sandboxes are always removed ────── + let found_a = filtered.iter().any(|s| s.object_name() == name_a); + let found_b = filtered.iter().any(|s| s.object_name() == name_b); + + if !found_a { + return Err(miette::miette!( + "sandbox '{name_a}' not found in label-filtered list (selector: {label_key}=labels-a)" + )); + } + if found_b { + return Err(miette::miette!( + "sandbox '{name_b}' appeared in label-filtered list but should have been excluded \ + (selector: {label_key}=labels-a)" + )); + } + + Ok(()) +} + +pub fn conformance_list(output: &str) -> Result<()> { + let scenarios = all_scenarios(); + + match output { + "json" => { + let values: Vec<_> = scenarios + .iter() + .map(|scenario| { + serde_json::json!({ + "name": scenario.name, + "description": scenario.description, + }) + }) + .collect(); + println!( + "{}", + serde_json::to_string_pretty(&values).into_diagnostic()? + ); + } + "yaml" => { + let values: Vec<_> = scenarios + .iter() + .map(|scenario| { + serde_json::json!({ + "name": scenario.name, + "description": scenario.description, + }) + }) + .collect(); + print!("{}", serde_yml::to_string(&values).into_diagnostic()?); + } + "table" => { + println!("{} conformance scenarios:", scenarios.len()); + for scenario in scenarios { + println!(" {:<20} {}", scenario.name, scenario.description); + } + } + _ => return Err(miette::miette!("unsupported output format: {output}")), + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::all_scenarios; + + #[test] + fn ci_scenario_set_contains_only_implemented_scenarios() { + let names = all_scenarios() + .iter() + .map(|scenario| scenario.name) + .collect::>(); + + assert_eq!( + names, + [ + "lifecycle", + "not-found", + "idempotent-delete", + "validate", + "concurrent", + "labels", + ] + ); + } +} diff --git a/e2e/rust/src/lib.rs b/e2e/rust/src/lib.rs index 144bc75084..f6ef66893d 100644 --- a/e2e/rust/src/lib.rs +++ b/e2e/rust/src/lib.rs @@ -9,4 +9,5 @@ //! - Parsing CLI output (ANSI stripping, field extraction) //! - TCP port utilities (wait for port, find free port) +pub mod conformance; pub mod harness; diff --git a/e2e/rust/tests/conformance.rs b/e2e/rust/tests/conformance.rs new file mode 100644 index 0000000000..57cbcbd32a --- /dev/null +++ b/e2e/rust/tests/conformance.rs @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Run the shared driver-agnostic conformance baseline against the gateway +//! registered by the surrounding e2e environment. + +use miette::Result; +use openshell_e2e::conformance::ConnectionOptions; +use std::path::PathBuf; + +#[tokio::test] +async fn gateway_conformance() -> Result<()> { + let gateway_endpoint = required_env("OPENSHELL_GATEWAY_ENDPOINT")?; + + openshell_e2e::conformance::conformance_run( + &gateway_endpoint, + &ConnectionOptions { + tls_ca: optional_env("OPENSHELL_CONFORMANCE_TLS_CA").map(PathBuf::from), + tls_cert: optional_env("OPENSHELL_CONFORMANCE_TLS_CERT").map(PathBuf::from), + tls_key: optional_env("OPENSHELL_CONFORMANCE_TLS_KEY").map(PathBuf::from), + }, + None, + conformance_timeout(), + "table", + ) + .await +} + +fn optional_env(name: &str) -> Option { + std::env::var(name) + .ok() + .filter(|value| !value.trim().is_empty()) +} + +fn required_env(name: &str) -> Result { + optional_env(name).ok_or_else(|| miette::miette!("{name} must be set by the e2e harness")) +} + +fn conformance_timeout() -> u64 { + std::env::var("OPENSHELL_CONFORMANCE_TIMEOUT") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(300) +} diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 64062b74d6..007bf7cb7e 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -528,6 +528,10 @@ e2e_register_mtls_gateway \ "${PKI_DIR}" export OPENSHELL_GATEWAY="${GATEWAY_NAME}" +export OPENSHELL_GATEWAY_ENDPOINT="${CLI_GATEWAY_ENDPOINT}" +export OPENSHELL_CONFORMANCE_TLS_CA="${PKI_DIR}/ca.crt" +export OPENSHELL_CONFORMANCE_TLS_CERT="${PKI_DIR}/client/tls.crt" +export OPENSHELL_CONFORMANCE_TLS_KEY="${PKI_DIR}/client/tls.key" export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-180}" echo "Waiting for gateway to become healthy..." diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index e4f47008c9..fb1291b1a9 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -363,6 +363,7 @@ run_scenario() { "${LOCAL_PORT}" export OPENSHELL_GATEWAY="${GATEWAY_NAME}" + export OPENSHELL_GATEWAY_ENDPOINT="${GATEWAY_ENDPOINT}" export OPENSHELL_E2E_DRIVER="kubernetes" export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" @@ -705,6 +706,7 @@ else "${LOCAL_PORT}" export OPENSHELL_GATEWAY="${GATEWAY_NAME}" + export OPENSHELL_GATEWAY_ENDPOINT="${GATEWAY_ENDPOINT}" export OPENSHELL_E2E_DRIVER="kubernetes" export OPENSHELL_E2E_SANDBOX_NAMESPACE="${NAMESPACE}" export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index 77b4c1324c..544483601e 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -467,6 +467,10 @@ e2e_register_mtls_gateway \ "${PKI_DIR}" export OPENSHELL_GATEWAY="${GATEWAY_NAME}" +export OPENSHELL_GATEWAY_ENDPOINT="${CLI_GATEWAY_ENDPOINT}" +export OPENSHELL_CONFORMANCE_TLS_CA="${PKI_DIR}/ca.crt" +export OPENSHELL_CONFORMANCE_TLS_CERT="${PKI_DIR}/client/tls.crt" +export OPENSHELL_CONFORMANCE_TLS_KEY="${PKI_DIR}/client/tls.key" export OPENSHELL_PROVISION_TIMEOUT="${OPENSHELL_PROVISION_TIMEOUT:-300}" echo "Waiting for gateway to become healthy..." diff --git a/tasks/test.toml b/tasks/test.toml index a75b05cce2..691d3a423c 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -58,10 +58,10 @@ run = "uv run pytest python/" hide = true ["e2e:rust"] -description = "Run Rust CLI e2e tests against a Docker-backed gateway" +description = "Run Rust e2e and API conformance tests against a Docker-backed gateway" run = [ "cargo build -p openshell-cli", - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker", + "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-api-conformance", ] ["e2e:websocket-conformance"] From 4557675fd6a8c0140eea95eb0adefe621745a5d7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 16 Jul 2026 15:02:11 +0200 Subject: [PATCH 2/7] test(e2e): define CLI conformance profile Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 8 ++++++-- e2e/rust/Cargo.toml | 11 +++++++++++ tasks/test.toml | 4 ++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md index aa885975cd..5a3510deb8 100644 --- a/e2e/rust/CONFORMANCE.md +++ b/e2e/rust/CONFORMANCE.md @@ -27,8 +27,12 @@ The capabilities scenario remains deferred until the gateway exposes driver capabilities through its public API. It is not included as a known-failing CI scenario. -The `e2e-api-conformance` profile invokes the scenario engine directly against -every gateway driver. +The test profiles separate the surface being validated: + +- `e2e-api-conformance` invokes the scenario engine directly against every + gateway driver. +- `e2e-cli-conformance` validates portable CLI behavior against the canonical + Docker-backed gateway. Gateway provisioners export the endpoint and any mTLS paths through `OPENSHELL_GATEWAY_ENDPOINT` and the `OPENSHELL_CONFORMANCE_TLS_*` variables. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index d94de47899..9ad554f1e0 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -18,6 +18,7 @@ publish = false [features] e2e = [] e2e-api-conformance = ["e2e"] +e2e-cli-conformance = ["e2e"] # Selects tests that rely on `host.openshell.internal` (the sandbox's stable # alias to the host running test fixtures). docker, podman, and vm wire the # alias unconditionally; the kube driver only does so when the chart's @@ -38,6 +39,16 @@ name = "conformance" path = "tests/conformance.rs" required-features = ["e2e-api-conformance"] +[[test]] +name = "sandbox_labels" +path = "tests/sandbox_labels.rs" +required-features = ["e2e-cli-conformance"] + +[[test]] +name = "sandbox_lifecycle" +path = "tests/sandbox_lifecycle.rs" +required-features = ["e2e-cli-conformance"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" diff --git a/tasks/test.toml b/tasks/test.toml index 691d3a423c..3626b25924 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -58,10 +58,10 @@ run = "uv run pytest python/" hide = true ["e2e:rust"] -description = "Run Rust e2e and API conformance tests against a Docker-backed gateway" +description = "Run Rust e2e, API conformance, and CLI conformance tests against a Docker-backed gateway" run = [ "cargo build -p openshell-cli", - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-api-conformance", + "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker,e2e-api-conformance,e2e-cli-conformance", ] ["e2e:websocket-conformance"] From ac71b13e5af17ae9083b6b4cefcfd5f8b4a57c80 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 17 Jul 2026 14:59:07 +0200 Subject: [PATCH 3/7] test(e2e): add command execution conformance Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 8 ++- e2e/rust/Cargo.toml | 10 +++ e2e/rust/e2e-docker.sh | 2 +- e2e/rust/e2e-vm.sh | 14 ++-- e2e/rust/src/conformance.rs | 105 +++++++++++++++++++++++++++- e2e/rust/tests/sandbox_lifecycle.rs | 37 +++++++++- e2e/rust/tests/smoke.rs | 46 +----------- e2e/rust/tests/vm_overlay.rs | 54 ++++++++++++++ tasks/test.toml | 4 +- 9 files changed, 217 insertions(+), 63 deletions(-) create mode 100644 e2e/rust/tests/vm_overlay.rs diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md index 5a3510deb8..42e2519b43 100644 --- a/e2e/rust/CONFORMANCE.md +++ b/e2e/rust/CONFORMANCE.md @@ -30,9 +30,13 @@ scenario. The test profiles separate the surface being validated: - `e2e-api-conformance` invokes the scenario engine directly against every - gateway driver. + gateway driver. This includes command execution through the streaming + `ExecSandbox` API. - `e2e-cli-conformance` validates portable CLI behavior against the canonical - Docker-backed gateway. + Docker-backed gateway, including the gateway smoke test. + +VM filesystem behavior is not driver-independent. The `vm_overlay` test remains +under the `e2e-vm` profile and runs alongside API conformance in the VM lane. Gateway provisioners export the endpoint and any mTLS paths through `OPENSHELL_GATEWAY_ENDPOINT` and the `OPENSHELL_CONFORMANCE_TLS_*` variables. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 9ad554f1e0..36fe92819f 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -49,6 +49,16 @@ name = "sandbox_lifecycle" path = "tests/sandbox_lifecycle.rs" required-features = ["e2e-cli-conformance"] +[[test]] +name = "smoke" +path = "tests/smoke.rs" +required-features = ["e2e-cli-conformance"] + +[[test]] +name = "vm_overlay" +path = "tests/vm_overlay.rs" +required-features = ["e2e-vm"] + [[test]] name = "custom_image" path = "tests/custom_image.rs" diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index 99cd6daf75..f2170fdfab 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -10,7 +10,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-smoke}" -E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e,e2e-docker}" +E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e-docker,e2e-cli-conformance}" DEFAULT_WORKLOAD_MANIFEST="${ROOT}/e2e/gpu/images/.build/workloads.yaml" cargo build -p openshell-cli diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index bfb85cd073..883dbdb59f 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -2,8 +2,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Run the Rust e2e smoke test against an openshell-gateway running the -# standalone VM compute driver (`openshell-driver-vm`). +# Run the Rust VM overlay test and API conformance suite against an +# openshell-gateway using the standalone VM compute driver. # # Architecture (post supervisor-initiated relay, PR #867): # * The gateway never dials the sandbox. Instead, the in-guest @@ -48,7 +48,7 @@ COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" -E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm,e2e-api-conformance}" +E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm,e2e-api-conformance,e2e-cli-conformance}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. # AF_UNIX SUN_LEN is 104 bytes on macOS (108 on Linux), so paths anchored @@ -59,7 +59,7 @@ E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm,e2e-api-conformance}" # so root state under `/tmp` unconditionally to keep UDS paths short. STATE_DIR_ROOT="/tmp" -# Smoke test timeouts. First boot extracts the embedded libkrun runtime +# E2E timeouts. First boot extracts the embedded libkrun runtime # (~60-90MB of zstd per architecture) and prepares an ext4 root disk from the # configured image. The guest then starts the sandbox supervisor directly; a cold # microVM is typically ready within ~15s after image preparation. @@ -246,7 +246,7 @@ printf '%s\n' "${GATEWAY_PID}" >"${GATEWAY_PID_FILE}" # # The gateway logs `INFO openshell_server: Server listening # address=0.0.0.0:` after its tonic listener is up. That is the -# only signal the smoke test needs — the VM driver is spawned eagerly +# only signal the e2e tests need — the VM driver is spawned eagerly # but sandboxes are created on demand, so "Server listening" is the # right gate here. @@ -267,7 +267,7 @@ done echo "==> Gateway ready after ${elapsed}s" -# ── Run the smoke test ─────────────────────────────────────────────── +# ── Run the selected test and API conformance ─────────────────────── # # The CLI uses the raw endpoint but still resolves matching metadata so it # can find the mTLS client bundle. @@ -284,7 +284,6 @@ export OPENSHELL_GATEWAY_ENDPOINT="${CLI_GATEWAY_ENDPOINT}" export OPENSHELL_CONFORMANCE_TLS_CA="${PKI_DIR}/ca.crt" export OPENSHELL_CONFORMANCE_TLS_CERT="${PKI_DIR}/client/tls.crt" export OPENSHELL_CONFORMANCE_TLS_KEY="${PKI_DIR}/client/tls.key" -export OPENSHELL_E2E_EXPECT_VM_OVERLAY=1 export OPENSHELL_E2E_DRIVER="vm" export OPENSHELL_E2E_VM_STATE_DIR="${RUN_STATE_DIR}" e2e_export_gateway_restart_metadata \ @@ -318,6 +317,7 @@ if [ -n "${E2E_TEST_OVERRIDE}" ]; then else run_e2e_test smoke run_e2e_test host_gateway_alias + run_e2e_test vm_overlay run_e2e_test vm_gateway_resume fi diff --git a/e2e/rust/src/conformance.rs b/e2e/rust/src/conformance.rs index 4172361e63..5138dd5259 100644 --- a/e2e/rust/src/conformance.rs +++ b/e2e/rust/src/conformance.rs @@ -10,9 +10,9 @@ use std::time::Duration; use futures_util::StreamExt; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::proto::{ - CreateSandboxRequest, DeleteSandboxRequest, GetSandboxRequest, GpuResourceRequirements, - ListSandboxesRequest, ResourceRequirements, SandboxPhase, SandboxSpec, WatchSandboxRequest, - open_shell_client::OpenShellClient, + CreateSandboxRequest, DeleteSandboxRequest, ExecSandboxRequest, GetSandboxRequest, + GpuResourceRequirements, ListSandboxesRequest, ResourceRequirements, SandboxPhase, SandboxSpec, + WatchSandboxRequest, exec_sandbox_event, open_shell_client::OpenShellClient, }; use openshell_core::{ObjectId, ObjectName}; use tonic::Code; @@ -121,6 +121,10 @@ fn all_scenarios() -> &'static [Scenario] { name: "labels", description: "Labels are persisted on create and filter list results correctly", }, + Scenario { + name: "exec", + description: "A command executes in a ready sandbox and streams its output and exit status", + }, ] } @@ -224,6 +228,7 @@ async fn run_scenario(name: &str, client: &mut GrpcClient, run_id: &str) -> Resu "validate" => scenario_validate(client).await, "concurrent" => scenario_concurrent(client, run_id).await, "labels" => scenario_labels(client, run_id).await, + "exec" => scenario_exec(client, run_id).await, _ => Err(miette::miette!("scenario '{name}' is not yet implemented")), } } @@ -703,6 +708,99 @@ async fn scenario_labels(client: &mut GrpcClient, run_id: &str) -> Result<()> { Ok(()) } +/// Scenario: execute a command in a ready sandbox through the gateway API. +/// +/// Verifies stdout and the final exit status independently, then deletes the +/// sandbox regardless of whether command execution succeeds. +async fn scenario_exec(client: &mut GrpcClient, run_id: &str) -> Result<()> { + const OUTPUT_MARKER: &str = "conformance-exec-ok"; + + let sandbox_name = format!("conformance-exec-{run_id}"); + let response = client + .create_sandbox(CreateSandboxRequest { + name: sandbox_name.clone(), + spec: Some(SandboxSpec::default()), + labels: HashMap::default(), + annotations: HashMap::default(), + }) + .await + .into_diagnostic() + .wrap_err("create_sandbox failed")?; + + let sandbox = response + .into_inner() + .sandbox + .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; + let sandbox_id = sandbox.object_id().to_string(); + + if let Err(error) = wait_for_ready(client, &sandbox_id, &sandbox_name).await { + let _ = client + .delete_sandbox(DeleteSandboxRequest { name: sandbox_name }) + .await; + return Err(error); + } + + let exec_result = async { + let mut stream = client + .exec_sandbox(ExecSandboxRequest { + sandbox_id, + command: vec![ + "sh".to_string(), + "-c".to_string(), + format!("printf {OUTPUT_MARKER}"), + ], + ..Default::default() + }) + .await + .into_diagnostic() + .wrap_err("exec_sandbox failed")? + .into_inner(); + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let mut exit_code = None; + + while let Some(event) = stream.next().await { + match event + .into_diagnostic() + .wrap_err("exec_sandbox stream error")? + .payload + { + Some(exec_sandbox_event::Payload::Stdout(chunk)) => stdout.extend(chunk.data), + Some(exec_sandbox_event::Payload::Stderr(chunk)) => stderr.extend(chunk.data), + Some(exec_sandbox_event::Payload::Exit(exit)) => exit_code = Some(exit.exit_code), + None => {} + } + } + + let stdout = String::from_utf8_lossy(&stdout); + let stderr = String::from_utf8_lossy(&stderr); + if !stdout.contains(OUTPUT_MARKER) { + return Err(miette::miette!( + "exec_sandbox stdout did not contain '{OUTPUT_MARKER}'; stdout={stdout:?}, stderr={stderr:?}" + )); + } + if exit_code != Some(0) { + return Err(miette::miette!( + "exec_sandbox returned exit code {exit_code:?}; stdout={stdout:?}, stderr={stderr:?}" + )); + } + + Ok(()) + } + .await; + + let cleanup_result = client + .delete_sandbox(DeleteSandboxRequest { name: sandbox_name }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox after exec failed"); + + exec_result?; + cleanup_result?; + Ok(()) +} + pub fn conformance_list(output: &str) -> Result<()> { let scenarios = all_scenarios(); @@ -765,6 +863,7 @@ mod tests { "validate", "concurrent", "labels", + "exec", ] ); } diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index 01e89422e0..9e4769bb20 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -67,6 +67,25 @@ async fn delete_sandbox(name: &str) { let _ = cmd.status().await; } +async fn exec_in_sandbox(name: &str, command: &[&str]) -> String { + let mut cmd = openshell_cmd(); + cmd.args(["sandbox", "exec", "--name", name, "--no-tty", "--"]) + .args(command) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = cmd.output().await.expect("spawn openshell sandbox exec"); + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).to_string(); + let combined = normalize_output(&format!("{stdout}{stderr}")); + assert!( + output.status.success(), + "sandbox exec should succeed (exit {:?}):\n{combined}", + output.status.code() + ); + combined +} + #[tokio::test] async fn sandbox_create_keeps_sandbox_after_tty_command_by_default() { let mut cmd = openshell_tty_cmd(&["sandbox", "create", "--", "echo", "OK"]); @@ -90,15 +109,27 @@ async fn sandbox_create_keeps_sandbox_after_tty_command_by_default() { let sandbox_name = extract_sandbox_name(&combined).expect("sandbox name should be present in output"); + let mut retained = false; for _ in 0..20 { if sandbox_list_names().await.contains(&sandbox_name) { - delete_sandbox(&sandbox_name).await; - return; + retained = true; + break; } sleep(Duration::from_millis(500)).await; } - panic!("sandbox {sandbox_name} should still exist by default"); + assert!( + retained, + "sandbox {sandbox_name} should still exist by default" + ); + + let exec_output = exec_in_sandbox(&sandbox_name, &["echo", "sandbox-exec-ok"]).await; + assert!( + exec_output.contains("sandbox-exec-ok"), + "expected sandbox exec output in:\n{exec_output}" + ); + + delete_sandbox(&sandbox_name).await; } #[tokio::test] diff --git a/e2e/rust/tests/smoke.rs b/e2e/rust/tests/smoke.rs index c27255e5ef..ab4817989a 100644 --- a/e2e/rust/tests/smoke.rs +++ b/e2e/rust/tests/smoke.rs @@ -6,9 +6,7 @@ //! Smoke test: verify the gateway is healthy, create a sandbox, exec a //! command inside it, and tear it down. //! -//! This test is cluster-agnostic — it works against any running gateway -//! (Docker-based cluster or openshell-driver-vm microVM). The `e2e:vm` mise -//! task uses it to validate the VM gateway after boot. +//! This test is cluster-agnostic and works against any running gateway. use std::process::Stdio; use std::time::Duration; @@ -68,10 +66,6 @@ async fn gateway_smoke() { sb.create_output, ); - if std::env::var_os("OPENSHELL_E2E_EXPECT_VM_OVERLAY").is_some() { - assert_vm_overlay_root(&sb.name).await; - } - // ── 3. Verify the sandbox appeared in the list ─────────────────── let mut list_cmd = openshell_cmd(); list_cmd @@ -99,41 +93,3 @@ async fn gateway_smoke() { // ── 4. Cleanup ─────────────────────────────────────────────────── sb.cleanup().await; } - -async fn assert_vm_overlay_root(sandbox_name: &str) { - let script = concat!( - "set -eu; ", - "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", - "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", - "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", - "if [ -e /opt/openshell/tls/tls.key ]; then ", - "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", - "fi; ", - "echo vm-overlay-ok", - ); - - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) - .arg("sh") - .arg("-lc") - .arg(script) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = exec_cmd - .output() - .await - .expect("failed to run VM overlay assertion"); - let combined = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )); - - assert!( - output.status.success() && combined.contains("vm-overlay-ok"), - "VM overlay assertion failed (status {:?}):\n{combined}", - output.status.code(), - ); -} diff --git a/e2e/rust/tests/vm_overlay.rs b/e2e/rust/tests/vm_overlay.rs new file mode 100644 index 0000000000..078c6f01de --- /dev/null +++ b/e2e/rust/tests/vm_overlay.rs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! VM-driver-specific assertions for the sandbox root filesystem. + +use std::process::Stdio; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::output::strip_ansi; +use openshell_e2e::harness::sandbox::SandboxGuard; + +#[tokio::test] +async fn vm_overlay() { + let mut sandbox = SandboxGuard::create(&["--", "echo", "vm-sandbox-ready"]) + .await + .expect("sandbox create should succeed"); + + let script = concat!( + "set -eu; ", + "test \"$(stat -f -c %T /)\" = \"overlayfs\"; ", + "printf \"overlay-write\\n\" > /sandbox/overlay-check; ", + "test \"$(cat /sandbox/overlay-check)\" = \"overlay-write\"; ", + "if [ -e /opt/openshell/tls/tls.key ]; then ", + "test \"$(stat -c %a /opt/openshell/tls/tls.key)\" = \"600\"; ", + "fi; ", + "echo vm-overlay-ok", + ); + + let mut exec_cmd = openshell_cmd(); + exec_cmd + .args(["sandbox", "exec", "--name", &sandbox.name, "--no-tty", "--"]) + .arg("sh") + .arg("-lc") + .arg(script) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = exec_cmd + .output() + .await + .expect("failed to run VM overlay assertion"); + let combined = strip_ansi(&format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )); + assert!( + output.status.success() && combined.contains("vm-overlay-ok"), + "VM overlay assertion failed (status {:?}):\n{combined}", + output.status.code(), + ); + + sandbox.cleanup().await; +} diff --git a/tasks/test.toml b/tasks/test.toml index 3626b25924..51434b57d5 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -125,11 +125,11 @@ env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } run = "e2e/rust/e2e-kubernetes.sh" ["e2e:vm"] -description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" +description = "Start openshell-gateway with the VM compute driver and run VM-specific and API conformance e2e tests" run = "e2e/rust/e2e-vm.sh" ["e2e:docker"] -description = "Run smoke e2e against a standalone gateway with the Docker compute driver" +description = "Run CLI smoke conformance against a standalone gateway with the Docker compute driver" run = "e2e/rust/e2e-docker.sh" ["e2e:mechanistic-smoke"] From 5dc83dd150586f05b5fb28476624417983e6ce24 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 17 Jul 2026 15:27:35 +0200 Subject: [PATCH 4/7] test(e2e): add process hardening API conformance Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 2 +- e2e/rust/src/conformance.rs | 174 +++++++++++++++++++------- e2e/rust/tests/core_dump_hardening.rs | 26 ---- 3 files changed, 127 insertions(+), 75 deletions(-) delete mode 100644 e2e/rust/tests/core_dump_hardening.rs diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md index 42e2519b43..fcdc250075 100644 --- a/e2e/rust/CONFORMANCE.md +++ b/e2e/rust/CONFORMANCE.md @@ -31,7 +31,7 @@ The test profiles separate the surface being validated: - `e2e-api-conformance` invokes the scenario engine directly against every gateway driver. This includes command execution through the streaming - `ExecSandbox` API. + `ExecSandbox` API and process-hardening checks. - `e2e-cli-conformance` validates portable CLI behavior against the canonical Docker-backed gateway, including the gateway smoke test. diff --git a/e2e/rust/src/conformance.rs b/e2e/rust/src/conformance.rs index 5138dd5259..dfbc0c6e8d 100644 --- a/e2e/rust/src/conformance.rs +++ b/e2e/rust/src/conformance.rs @@ -125,6 +125,10 @@ fn all_scenarios() -> &'static [Scenario] { name: "exec", description: "A command executes in a ready sandbox and streams its output and exit status", }, + Scenario { + name: "process-hardening", + description: "Sandbox processes start with core dumps disabled", + }, ] } @@ -229,6 +233,7 @@ async fn run_scenario(name: &str, client: &mut GrpcClient, run_id: &str) -> Resu "concurrent" => scenario_concurrent(client, run_id).await, "labels" => scenario_labels(client, run_id).await, "exec" => scenario_exec(client, run_id).await, + "process-hardening" => scenario_process_hardening(client, run_id).await, _ => Err(miette::miette!("scenario '{name}' is not yet implemented")), } } @@ -708,17 +713,56 @@ async fn scenario_labels(client: &mut GrpcClient, run_id: &str) -> Result<()> { Ok(()) } -/// Scenario: execute a command in a ready sandbox through the gateway API. -/// -/// Verifies stdout and the final exit status independently, then deletes the -/// sandbox regardless of whether command execution succeeds. -async fn scenario_exec(client: &mut GrpcClient, run_id: &str) -> Result<()> { - const OUTPUT_MARKER: &str = "conformance-exec-ok"; +struct ExecResult { + stdout: String, + stderr: String, + exit_code: Option, +} - let sandbox_name = format!("conformance-exec-{run_id}"); +async fn exec_command( + client: &mut GrpcClient, + sandbox_id: &str, + command: Vec, +) -> Result { + let mut stream = client + .exec_sandbox(ExecSandboxRequest { + sandbox_id: sandbox_id.to_string(), + command, + ..Default::default() + }) + .await + .into_diagnostic() + .wrap_err("exec_sandbox failed")? + .into_inner(); + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + let mut exit_code = None; + + while let Some(event) = stream.next().await { + match event + .into_diagnostic() + .wrap_err("exec_sandbox stream error")? + .payload + { + Some(exec_sandbox_event::Payload::Stdout(chunk)) => stdout.extend(chunk.data), + Some(exec_sandbox_event::Payload::Stderr(chunk)) => stderr.extend(chunk.data), + Some(exec_sandbox_event::Payload::Exit(exit)) => exit_code = Some(exit.exit_code), + None => {} + } + } + + Ok(ExecResult { + stdout: String::from_utf8_lossy(&stdout).into_owned(), + stderr: String::from_utf8_lossy(&stderr).into_owned(), + exit_code, + }) +} + +async fn create_ready_sandbox(client: &mut GrpcClient, sandbox_name: &str) -> Result { let response = client .create_sandbox(CreateSandboxRequest { - name: sandbox_name.clone(), + name: sandbox_name.to_string(), spec: Some(SandboxSpec::default()), labels: HashMap::default(), annotations: HashMap::default(), @@ -733,56 +777,52 @@ async fn scenario_exec(client: &mut GrpcClient, run_id: &str) -> Result<()> { .ok_or_else(|| miette::miette!("create_sandbox response missing sandbox"))?; let sandbox_id = sandbox.object_id().to_string(); - if let Err(error) = wait_for_ready(client, &sandbox_id, &sandbox_name).await { + if let Err(error) = wait_for_ready(client, &sandbox_id, sandbox_name).await { let _ = client - .delete_sandbox(DeleteSandboxRequest { name: sandbox_name }) + .delete_sandbox(DeleteSandboxRequest { + name: sandbox_name.to_string(), + }) .await; return Err(error); } - let exec_result = async { - let mut stream = client - .exec_sandbox(ExecSandboxRequest { - sandbox_id, - command: vec![ - "sh".to_string(), - "-c".to_string(), - format!("printf {OUTPUT_MARKER}"), - ], - ..Default::default() - }) - .await - .into_diagnostic() - .wrap_err("exec_sandbox failed")? - .into_inner(); - - let mut stdout = Vec::new(); - let mut stderr = Vec::new(); - let mut exit_code = None; - - while let Some(event) = stream.next().await { - match event - .into_diagnostic() - .wrap_err("exec_sandbox stream error")? - .payload - { - Some(exec_sandbox_event::Payload::Stdout(chunk)) => stdout.extend(chunk.data), - Some(exec_sandbox_event::Payload::Stderr(chunk)) => stderr.extend(chunk.data), - Some(exec_sandbox_event::Payload::Exit(exit)) => exit_code = Some(exit.exit_code), - None => {} - } - } + Ok(sandbox_id) +} + +/// Scenario: execute a command in a ready sandbox through the gateway API. +/// +/// Verifies stdout and the final exit status independently, then deletes the +/// sandbox regardless of whether command execution succeeds. +async fn scenario_exec(client: &mut GrpcClient, run_id: &str) -> Result<()> { + const OUTPUT_MARKER: &str = "conformance-exec-ok"; + + let sandbox_name = format!("conformance-exec-{run_id}"); + let sandbox_id = create_ready_sandbox(client, &sandbox_name).await?; - let stdout = String::from_utf8_lossy(&stdout); - let stderr = String::from_utf8_lossy(&stderr); - if !stdout.contains(OUTPUT_MARKER) { + let exec_result = async { + let result = exec_command( + client, + &sandbox_id, + vec![ + "sh".to_string(), + "-c".to_string(), + format!("printf {OUTPUT_MARKER}"), + ], + ) + .await?; + if !result.stdout.contains(OUTPUT_MARKER) { return Err(miette::miette!( - "exec_sandbox stdout did not contain '{OUTPUT_MARKER}'; stdout={stdout:?}, stderr={stderr:?}" + "exec_sandbox stdout did not contain '{OUTPUT_MARKER}'; stdout={:?}, stderr={:?}", + result.stdout, + result.stderr, )); } - if exit_code != Some(0) { + if result.exit_code != Some(0) { return Err(miette::miette!( - "exec_sandbox returned exit code {exit_code:?}; stdout={stdout:?}, stderr={stderr:?}" + "exec_sandbox returned exit code {:?}; stdout={:?}, stderr={:?}", + result.exit_code, + result.stdout, + result.stderr, )); } @@ -801,6 +841,43 @@ async fn scenario_exec(client: &mut GrpcClient, run_id: &str) -> Result<()> { Ok(()) } +/// Scenario: sandbox processes start with core dumps disabled. +async fn scenario_process_hardening(client: &mut GrpcClient, run_id: &str) -> Result<()> { + const OUTPUT_MARKER: &str = "core-limit-ok"; + + let sandbox_name = format!("conformance-process-hardening-{run_id}"); + let sandbox_id = create_ready_sandbox(client, &sandbox_name).await?; + let command_result = exec_command( + client, + &sandbox_id, + vec![ + "sh".to_string(), + "-lc".to_string(), + format!("test \"$(ulimit -c)\" = 0 && printf {OUTPUT_MARKER}"), + ], + ) + .await; + + let cleanup_result = client + .delete_sandbox(DeleteSandboxRequest { name: sandbox_name }) + .await + .into_diagnostic() + .wrap_err("delete_sandbox after process hardening check failed"); + + let result = command_result?; + cleanup_result?; + if result.exit_code != Some(0) || !result.stdout.contains(OUTPUT_MARKER) { + return Err(miette::miette!( + "sandbox process core-dump check failed with exit code {:?}; stdout={:?}, stderr={:?}", + result.exit_code, + result.stdout, + result.stderr, + )); + } + + Ok(()) +} + pub fn conformance_list(output: &str) -> Result<()> { let scenarios = all_scenarios(); @@ -864,6 +941,7 @@ mod tests { "concurrent", "labels", "exec", + "process-hardening", ] ); } diff --git a/e2e/rust/tests/core_dump_hardening.rs b/e2e/rust/tests/core_dump_hardening.rs deleted file mode 100644 index a400b6c218..0000000000 --- a/e2e/rust/tests/core_dump_hardening.rs +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -#[tokio::test] -async fn sandbox_processes_disable_core_dumps() { - let mut sb = SandboxGuard::create(&[ - "--", - "sh", - "-lc", - "test \"$(ulimit -c)\" = 0 && echo core-limit-ok", - ]) - .await - .expect("sandbox create should succeed"); - - assert!( - sb.create_output.contains("core-limit-ok"), - "expected sandbox output to confirm core dumps are disabled:\n{}", - sb.create_output, - ); - - sb.cleanup().await; -} From d2206ee604fd9a0e52e3c5dfe94654ea04b84c6f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 17 Jul 2026 15:27:49 +0200 Subject: [PATCH 5/7] test(e2e): classify portable CLI workflows Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 3 ++- e2e/rust/Cargo.toml | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md index fcdc250075..11c8852ce5 100644 --- a/e2e/rust/CONFORMANCE.md +++ b/e2e/rust/CONFORMANCE.md @@ -33,7 +33,8 @@ The test profiles separate the surface being validated: gateway driver. This includes command execution through the streaming `ExecSandbox` API and process-hardening checks. - `e2e-cli-conformance` validates portable CLI behavior against the canonical - Docker-backed gateway, including the gateway smoke test. + Docker-backed gateway, including the gateway smoke test, port forwarding, + and file upload/download workflows. VM filesystem behavior is not driver-independent. The `vm_overlay` test remains under the `e2e-vm` profile and runs alongside API conformance in the VM lane. diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 36fe92819f..7f2444a704 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -49,6 +49,21 @@ name = "sandbox_lifecycle" path = "tests/sandbox_lifecycle.rs" required-features = ["e2e-cli-conformance"] +[[test]] +name = "port_forward" +path = "tests/port_forward.rs" +required-features = ["e2e-cli-conformance"] + +[[test]] +name = "sync" +path = "tests/sync.rs" +required-features = ["e2e-cli-conformance"] + +[[test]] +name = "upload_create" +path = "tests/upload_create.rs" +required-features = ["e2e-cli-conformance"] + [[test]] name = "smoke" path = "tests/smoke.rs" From 9358c6640f2ac0524d2f309ab62be9e0e3764ab1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 17 Jul 2026 15:28:03 +0200 Subject: [PATCH 6/7] refactor(e2e): share gateway resume scenario Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 4 + e2e/rust/src/harness/mod.rs | 1 + e2e/rust/src/harness/resume.rs | 118 ++++++++++++++++++++++++ e2e/rust/tests/gateway_resume.rs | 93 +++++++------------ e2e/rust/tests/podman_gateway_resume.rs | 54 +---------- e2e/rust/tests/vm_gateway_resume.rs | 64 +++---------- 6 files changed, 170 insertions(+), 164 deletions(-) create mode 100644 e2e/rust/src/harness/resume.rs diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md index 11c8852ce5..ac046c0ccb 100644 --- a/e2e/rust/CONFORMANCE.md +++ b/e2e/rust/CONFORMANCE.md @@ -39,6 +39,10 @@ The test profiles separate the surface being validated: VM filesystem behavior is not driver-independent. The `vm_overlay` test remains under the `e2e-vm` profile and runs alongside API conformance in the VM lane. +Gateway restart coverage uses a shared application-level resume scenario with +driver-specific observation hooks. It remains outside the baseline conformance +profiles because restarting a gateway requires control from its provisioner. + Gateway provisioners export the endpoint and any mTLS paths through `OPENSHELL_GATEWAY_ENDPOINT` and the `OPENSHELL_CONFORMANCE_TLS_*` variables. Set `OPENSHELL_CONFORMANCE_TIMEOUT` to change the per-scenario timeout. diff --git a/e2e/rust/src/harness/mod.rs b/e2e/rust/src/harness/mod.rs index f2dfd5ec9c..9fe0446d81 100644 --- a/e2e/rust/src/harness/mod.rs +++ b/e2e/rust/src/harness/mod.rs @@ -9,4 +9,5 @@ pub mod container; pub mod gateway; pub mod output; pub mod port; +pub mod resume; pub mod sandbox; diff --git a/e2e/rust/src/harness/resume.rs b/e2e/rust/src/harness/resume.rs new file mode 100644 index 0000000000..2c7addd851 --- /dev/null +++ b/e2e/rust/src/harness/resume.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared application-level contract for gateway restart and sandbox resume. + +use std::time::Duration; + +use super::cli::{sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains}; +use super::gateway::ManagedGateway; +use super::sandbox::SandboxGuard; + +const READY_MARKER: &str = "gateway-resume-ready"; +const RESUME_FILE: &str = "/sandbox/gateway-resume-state"; + +/// Driver-specific observations around the shared gateway restart sequence. +#[allow(async_fn_in_trait)] // Internal e2e hook; callers do not require Send bounds. +pub trait GatewayResumeHooks { + fn flush_state_before_ready(&self) -> bool { + false + } + + async fn before_gateway_stop(&self, _sandbox_name: &str) -> Result<(), String> { + Ok(()) + } + + async fn after_gateway_stop(&self, _sandbox_name: &str) -> Result<(), String> { + Ok(()) + } + + async fn after_gateway_start(&self, _sandbox_name: &str) -> Result<(), String> { + Ok(()) + } +} + +/// Hooks for drivers that need only the application-level resume assertions. +pub struct NoopGatewayResumeHooks; + +impl GatewayResumeHooks for NoopGatewayResumeHooks {} + +/// Run the common gateway restart and sandbox state-preservation contract. +pub async fn run_gateway_resume_scenario( + gateway: &ManagedGateway, + driver_name: &str, + hooks: &impl GatewayResumeHooks, +) { + wait_for_healthy(Duration::from_secs(30)) + .await + .unwrap_or_else(|error| panic!("{driver_name} gateway should start healthy: {error}")); + + let flush = if hooks.flush_state_before_ready() { + "sync; " + } else { + "" + }; + let script = format!( + "echo before-restart > {RESUME_FILE}; {flush}echo {READY_MARKER}; while true; do sleep 1; done" + ); + let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) + .await + .unwrap_or_else(|error| panic!("create long-running {driver_name} sandbox: {error}")); + + let before_restart = sandbox + .exec(&["cat", RESUME_FILE]) + .await + .unwrap_or_else(|error| panic!("read {driver_name} sandbox state before restart: {error}")); + assert!( + before_restart.contains("before-restart"), + "{driver_name} sandbox state was not written before restart:\n{before_restart}" + ); + + hooks + .before_gateway_stop(&sandbox.name) + .await + .unwrap_or_else(|error| panic!("{driver_name} pre-stop observation failed: {error}")); + gateway + .stop() + .unwrap_or_else(|error| panic!("stop {driver_name} e2e gateway: {error}")); + hooks + .after_gateway_stop(&sandbox.name) + .await + .unwrap_or_else(|error| panic!("{driver_name} post-stop observation failed: {error}")); + + gateway + .start() + .unwrap_or_else(|error| panic!("restart {driver_name} e2e gateway: {error}")); + wait_for_healthy(Duration::from_secs(120)) + .await + .unwrap_or_else(|error| { + panic!("{driver_name} gateway should become healthy after restart: {error}") + }); + hooks + .after_gateway_start(&sandbox.name) + .await + .unwrap_or_else(|error| panic!("{driver_name} post-start observation failed: {error}")); + + let names = sandbox_names() + .await + .unwrap_or_else(|error| panic!("list {driver_name} sandboxes after restart: {error}")); + assert!( + names.contains(&sandbox.name), + "{} sandbox '{}' should still be listed after gateway restart. Names: {names:?}", + driver_name, + sandbox.name, + ); + + wait_for_sandbox_exec_contains( + &sandbox.name, + &["cat", RESUME_FILE], + "before-restart", + Duration::from_secs(240), + ) + .await + .unwrap_or_else(|error| { + panic!("{driver_name} sandbox should resume with its state preserved: {error}") + }); + + sandbox.cleanup().await; +} diff --git a/e2e/rust/tests/gateway_resume.rs b/e2e/rust/tests/gateway_resume.rs index 8f850e485d..152a67eda7 100644 --- a/e2e/rust/tests/gateway_resume.rs +++ b/e2e/rust/tests/gateway_resume.rs @@ -12,16 +12,11 @@ use std::process::{Command, Stdio}; use std::time::Duration; -use openshell_e2e::harness::cli::{ - sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains, -}; use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::sandbox::SandboxGuard; +use openshell_e2e::harness::resume::{GatewayResumeHooks, run_gateway_resume_scenario}; use tokio::time::sleep; const MANAGED_BY_LABEL_FILTER: &str = "label=openshell.ai/managed-by=openshell"; -const READY_MARKER: &str = "gateway-resume-ready"; -const RESUME_FILE: &str = "/sandbox/gateway-resume-state"; const SANDBOX_NAMESPACE_LABEL: &str = "openshell.ai/sandbox-namespace"; const SANDBOX_NAME_LABEL: &str = "openshell.ai/sandbox-name"; @@ -116,6 +111,32 @@ async fn wait_for_container_running( } } +struct DockerResumeHooks<'a> { + namespace: &'a str, +} + +impl GatewayResumeHooks for DockerResumeHooks<'_> { + async fn before_gateway_stop(&self, sandbox_name: &str) -> Result<(), String> { + wait_for_container_running(self.namespace, sandbox_name, true, Duration::from_secs(60)) + .await + } + + async fn after_gateway_stop(&self, sandbox_name: &str) -> Result<(), String> { + wait_for_container_running( + self.namespace, + sandbox_name, + false, + Duration::from_secs(120), + ) + .await + } + + async fn after_gateway_start(&self, sandbox_name: &str) -> Result<(), String> { + wait_for_container_running(self.namespace, sandbox_name, true, Duration::from_secs(120)) + .await + } +} + #[tokio::test] async fn docker_gateway_restart_resumes_running_sandbox() { let Some(gateway) = ManagedGateway::from_env().expect("load managed e2e gateway metadata") @@ -131,58 +152,12 @@ async fn docker_gateway_restart_resumes_running_sandbox() { return; }; - wait_for_healthy(Duration::from_secs(30)) - .await - .expect("gateway should start healthy"); - - let script = format!( - "echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done" - ); - let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) - .await - .expect("create long-running sandbox"); - - let before_restart = sandbox - .exec(&["cat", RESUME_FILE]) - .await - .expect("read sandbox state before restart"); - assert!( - before_restart.contains("before-restart"), - "sandbox state was not written before restart:\n{before_restart}" - ); - - wait_for_container_running(&namespace, &sandbox.name, true, Duration::from_secs(60)) - .await - .expect("sandbox container should be running before gateway restart"); - - gateway.stop().expect("stop e2e gateway"); - wait_for_container_running(&namespace, &sandbox.name, false, Duration::from_secs(120)) - .await - .expect("gateway shutdown should stop managed Docker sandboxes"); - - gateway.start().expect("restart e2e gateway"); - wait_for_healthy(Duration::from_secs(120)) - .await - .expect("gateway should become healthy after restart"); - wait_for_container_running(&namespace, &sandbox.name, true, Duration::from_secs(120)) - .await - .expect("gateway startup should resume the Docker sandbox container"); - - let names = sandbox_names().await.expect("list sandboxes after restart"); - assert!( - names.contains(&sandbox.name), - "sandbox '{}' should still be listed after gateway restart. Names: {names:?}", - sandbox.name - ); - - wait_for_sandbox_exec_contains( - &sandbox.name, - &["cat", RESUME_FILE], - "before-restart", - Duration::from_secs(240), + run_gateway_resume_scenario( + &gateway, + "Docker", + &DockerResumeHooks { + namespace: &namespace, + }, ) - .await - .expect("sandbox should become ready again with its state preserved"); - - sandbox.cleanup().await; + .await; } diff --git a/e2e/rust/tests/podman_gateway_resume.rs b/e2e/rust/tests/podman_gateway_resume.rs index 2600202537..82b2c7ca56 100644 --- a/e2e/rust/tests/podman_gateway_resume.rs +++ b/e2e/rust/tests/podman_gateway_resume.rs @@ -12,16 +12,8 @@ //! pattern: verify sandbox survival at the application level without asserting //! intermediate container-state transitions. -use std::time::Duration; - -use openshell_e2e::harness::cli::{ - sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains, -}; use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::sandbox::SandboxGuard; - -const READY_MARKER: &str = "podman-gateway-resume-ready"; -const RESUME_FILE: &str = "/sandbox/podman-gateway-resume-state"; +use openshell_e2e::harness::resume::{NoopGatewayResumeHooks, run_gateway_resume_scenario}; #[tokio::test] async fn podman_gateway_restart_resumes_running_sandbox() { @@ -37,47 +29,5 @@ async fn podman_gateway_restart_resumes_running_sandbox() { return; }; - wait_for_healthy(Duration::from_secs(30)) - .await - .expect("gateway should start healthy"); - - let script = format!( - "echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done" - ); - let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) - .await - .expect("create long-running Podman sandbox"); - - let before_restart = sandbox - .exec(&["cat", RESUME_FILE]) - .await - .expect("read Podman sandbox state before restart"); - assert!( - before_restart.contains("before-restart"), - "sandbox state was not written before restart:\n{before_restart}" - ); - - gateway.stop().expect("stop e2e gateway"); - gateway.start().expect("restart e2e gateway"); - wait_for_healthy(Duration::from_secs(120)) - .await - .expect("gateway should become healthy after restart"); - - let names = sandbox_names().await.expect("list sandboxes after restart"); - assert!( - names.contains(&sandbox.name), - "sandbox '{}' should still be listed after gateway restart. Names: {names:?}", - sandbox.name - ); - - wait_for_sandbox_exec_contains( - &sandbox.name, - &["cat", RESUME_FILE], - "before-restart", - Duration::from_secs(240), - ) - .await - .expect("Podman sandbox should become ready again with its state preserved"); - - sandbox.cleanup().await; + run_gateway_resume_scenario(&gateway, "Podman", &NoopGatewayResumeHooks).await; } diff --git a/e2e/rust/tests/vm_gateway_resume.rs b/e2e/rust/tests/vm_gateway_resume.rs index 4c502bb9c9..870a2099ab 100644 --- a/e2e/rust/tests/vm_gateway_resume.rs +++ b/e2e/rust/tests/vm_gateway_resume.rs @@ -9,16 +9,16 @@ //! This test is gated behind the `e2e-vm` feature because it requires the VM //! driver runtime prepared by `e2e/rust/e2e-vm.sh`. -use std::time::Duration; - -use openshell_e2e::harness::cli::{ - sandbox_names, wait_for_healthy, wait_for_sandbox_exec_contains, -}; use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::sandbox::SandboxGuard; +use openshell_e2e::harness::resume::{GatewayResumeHooks, run_gateway_resume_scenario}; -const READY_MARKER: &str = "vm-gateway-resume-ready"; -const RESUME_FILE: &str = "/sandbox/vm-gateway-resume-state"; +struct VmResumeHooks; + +impl GatewayResumeHooks for VmResumeHooks { + fn flush_state_before_ready(&self) -> bool { + true + } +} #[tokio::test] async fn vm_gateway_restart_resumes_running_sandbox() { @@ -32,50 +32,8 @@ async fn vm_gateway_restart_resumes_running_sandbox() { return; }; - wait_for_healthy(Duration::from_secs(30)) - .await - .expect("gateway should start healthy"); - // The gateway restart terminates the VM process before re-adopting its - // overlay. Flush the marker before reporting readiness so the assertion - // verifies durable overlay state rather than guest page-cache timing. - let script = format!( - "echo before-restart > {RESUME_FILE}; sync; echo {READY_MARKER}; while true; do sleep 1; done" - ); - let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) - .await - .expect("create long-running VM sandbox"); - - let before_restart = sandbox - .exec(&["cat", RESUME_FILE]) - .await - .expect("read VM sandbox state before restart"); - assert!( - before_restart.contains("before-restart"), - "VM sandbox state was not written before restart:\n{before_restart}" - ); - - gateway.stop().expect("stop e2e gateway"); - gateway.start().expect("restart e2e gateway"); - wait_for_healthy(Duration::from_secs(120)) - .await - .expect("gateway should become healthy after restart"); - - let names = sandbox_names().await.expect("list sandboxes after restart"); - assert!( - names.contains(&sandbox.name), - "sandbox '{}' should still be listed after gateway restart. Names: {names:?}", - sandbox.name - ); - - wait_for_sandbox_exec_contains( - &sandbox.name, - &["cat", RESUME_FILE], - "before-restart", - Duration::from_secs(240), - ) - .await - .expect("VM sandbox should become ready again with its overlay state preserved"); - - sandbox.cleanup().await; + // overlay. The VM hook flushes the marker before reporting readiness so + // the assertion verifies durable overlay state rather than page-cache timing. + run_gateway_resume_scenario(&gateway, "VM", &VmResumeHooks).await; } From 6fe2c16eb9da14283d7eea8d4b7b13f2501b12c6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 17 Jul 2026 16:09:26 +0200 Subject: [PATCH 7/7] docs(e2e): diagram conformance test architecture Signed-off-by: Evan Lezar --- e2e/rust/CONFORMANCE.md | 70 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/e2e/rust/CONFORMANCE.md b/e2e/rust/CONFORMANCE.md index ac046c0ccb..c975993e75 100644 --- a/e2e/rust/CONFORMANCE.md +++ b/e2e/rust/CONFORMANCE.md @@ -36,6 +36,76 @@ The test profiles separate the surface being validated: Docker-backed gateway, including the gateway smoke test, port forwarding, and file upload/download workflows. +## Test architecture + +```mermaid +flowchart LR + subgraph Provisioning["Gateway instantiation"] + P["Provisioner
Docker · Podman · Kubernetes · VM"] + C["Connection context
endpoint · mTLS · gateway registration"] + L["Lifecycle control
managed gateway metadata"] + end + + subgraph Tests["Test execution"] + API["API conformance
direct gRPC scenarios"] + CLI["CLI conformance
portable user workflows"] + DS["Driver-specific tests
runtime and infrastructure assertions"] + RR["Shared restart/resume scenario
driver-specific hooks"] + end + + OCLI["openshell CLI"] + GW["OpenShell gateway"] + DR["Compute driver
Docker · Podman · Kubernetes · VM"] + SB["Sandbox supervisor
and runtime"] + + P -.->|"instantiates"| GW + P -.->|"configures"| DR + P --> C + P --> L + + C -.-> API + C -.-> CLI + C -.-> DS + C -.-> RR + L -.-> RR + + API -->|"tonic / gRPC"| GW + CLI --> OCLI + DS --> OCLI + RR -->|"application assertions"| OCLI + + OCLI -->|"gateway API"| GW + GW --> DR + DR --> SB + + DS -.->|"driver or runtime observations"| DR + DS -.->|"sandbox-specific assertions"| SB + RR -.->|"stop / start"| GW +``` + +Solid arrows show normal request paths. Dashed arrows show provisioning, +connection metadata, lifecycle control, or implementation-specific +observations. Update this diagram whenever a test moves between profiles, a +new test surface is added, or the boundary between provisioning and test +execution changes. + +## Driver coverage + +| Driver | API conformance | CLI conformance | Driver-specific intent | Possible evolution | +| --- | --- | --- | --- | --- | +| Docker | Full baseline | Canonical full CLI profile; the focused task runs smoke only | Custom images, Docker preflight, volumes, restart/resume, and host gateway | Keep as the canonical CLI lane; external-image checks could become optional CLI conformance | +| Podman | Full baseline | Not currently enabled | Podman re-adoption, volumes, token restart, and host gateway | Enable selected CLI conformance to prove portability; keep restart behavior operational | +| Kubernetes | Full baseline | Not currently enabled | Readiness, user namespaces, Kubernetes topology, and host gateway | Enable selected CLI conformance where cluster fixtures permit; keep pod and deployment assertions driver-specific | +| VM | Full baseline | Smoke workflow | Overlay persistence, TLS permissions, host gateway, and restart/resume | Expand selected CLI conformance; keep overlay assertions driver-specific and restart behavior operational | + +Potential cross-driver follow-ups include: + +- API security and network conformance for bypass detection and `NO_PROXY` +- CLI conformance for settings management, live policy updates, and provider + auto-creation +- optional CLI conformance for external community-image resolution +- operational conformance for provisioner-controlled gateway restart/resume + VM filesystem behavior is not driver-independent. The `vm_overlay` test remains under the `e2e-vm` profile and runs alongside API conformance in the VM lane.