From 62b03f00588eb2dcfa45deb950a66666603b6b95 Mon Sep 17 00:00:00 2001 From: "Huabing (Robin) Zhao" Date: Thu, 25 Jun 2026 01:11:01 +0800 Subject: [PATCH 01/20] fix(docs): add step for creating the GatewayClass (#1984) Signed-off-by: Huabing (Robin) Zhao --- docs/kubernetes/ingress.mdx | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 3ed9a4cd5a..a476370735 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -14,17 +14,32 @@ OpenShell uses the [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io) for ## Install Envoy Gateway -Envoy Gateway installs the Gateway API CRDs and registers the `eg` GatewayClass: +Envoy Gateway installs the Gateway API CRDs and controller: ```shell helm install eg \ oci://docker.io/envoyproxy/gateway-helm \ - --version v1.7.2 \ + --version v1.8.1 \ --namespace envoy-gateway-system \ --create-namespace \ --wait ``` +## Create the GatewayClass + +Create the `eg` GatewayClass that the OpenShell chart references: + +```shell +kubectl apply -f - <<'EOF' +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +EOF +``` + Verify the GatewayClass is accepted: ```shell From c7879a0a1d0a1d914dc745e843255ce1141cda1e Mon Sep 17 00:00:00 2001 From: Simon Scatton <44714756+SDAChess@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:08:04 +0200 Subject: [PATCH 02/20] test(e2e): stop using custom e2e binary builds (#2000) Remove the production-crate dev-settings feature so e2e tests build the same gateway, CLI, and supervisor binaries that we ship. Move settings coverage from dummy test-only keys to the production ocsf_json_enabled setting. Signed-off-by: Simon Scatton --- .github/workflows/docker-build.yml | 4 +- .github/workflows/rust-native-build.yml | 2 +- crates/openshell-cli/Cargo.toml | 1 - crates/openshell-cli/src/run.rs | 20 +---- crates/openshell-core/Cargo.toml | 4 - crates/openshell-core/src/settings.rs | 23 ------ crates/openshell-server/Cargo.toml | 1 - crates/openshell-server/src/grpc/policy.rs | 94 ++++++++++++---------- e2e/rust/e2e-docker.sh | 2 +- e2e/rust/e2e-kubernetes.sh | 2 +- e2e/rust/e2e-podman.sh | 2 +- e2e/rust/e2e-vm.sh | 3 +- e2e/rust/tests/settings_management.rs | 6 +- e2e/support/gateway-common.sh | 6 +- tasks/scripts/stage-prebuilt-binaries.sh | 2 +- tasks/test.toml | 6 +- 16 files changed, 73 insertions(+), 105 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index aae0a05b9b..4c92bc3857 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -84,13 +84,13 @@ jobs: gateway) binary_component=gateway binary_name=openshell-gateway - features="openshell-core/dev-settings bundled-z3" + features="bundled-z3" has_image=true ;; supervisor) binary_component=sandbox binary_name=openshell-sandbox - features="openshell-core/dev-settings" + features="" has_image=true ;; cli) diff --git a/.github/workflows/rust-native-build.yml b/.github/workflows/rust-native-build.yml index 65d691e39c..89993ba12d 100644 --- a/.github/workflows/rust-native-build.yml +++ b/.github/workflows/rust-native-build.yml @@ -30,7 +30,7 @@ on: description: "Cargo features to enable" required: false type: string - default: "openshell-core/dev-settings" + default: "" retention-days: description: "Artifact retention period" required: false diff --git a/crates/openshell-cli/Cargo.toml b/crates/openshell-cli/Cargo.toml index 577fb73b93..8e4cb3fb25 100644 --- a/crates/openshell-cli/Cargo.toml +++ b/crates/openshell-cli/Cargo.toml @@ -86,7 +86,6 @@ workspace = true [features] bundled-z3 = ["openshell-prover/bundled-z3"] -dev-settings = ["openshell-core/dev-settings"] [dev-dependencies] futures = { workspace = true } diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 1c3fd8a820..29468aa49f 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -8097,16 +8097,15 @@ mod tests { )); } - #[cfg(feature = "dev-settings")] #[test] fn parse_cli_setting_value_parses_bool_aliases() { - let yes_value = parse_cli_setting_value("dummy_bool", "yes").expect("parse yes"); + let yes_value = parse_cli_setting_value("ocsf_json_enabled", "yes").expect("parse yes"); assert_eq!( yes_value.value, Some(openshell_core::proto::setting_value::Value::BoolValue(true)) ); - let zero_value = parse_cli_setting_value("dummy_bool", "0").expect("parse 0"); + let zero_value = parse_cli_setting_value("ocsf_json_enabled", "0").expect("parse 0"); assert_eq!( zero_value.value, Some(openshell_core::proto::setting_value::Value::BoolValue( @@ -8115,21 +8114,10 @@ mod tests { ); } - #[cfg(feature = "dev-settings")] - #[test] - fn parse_cli_setting_value_parses_int_key() { - let int_value = parse_cli_setting_value("dummy_int", "42").expect("parse int"); - assert_eq!( - int_value.value, - Some(openshell_core::proto::setting_value::Value::IntValue(42)) - ); - } - - #[cfg(feature = "dev-settings")] #[test] fn parse_cli_setting_value_rejects_invalid_bool() { - let err = - parse_cli_setting_value("dummy_bool", "maybe").expect_err("invalid bool should fail"); + let err = parse_cli_setting_value("ocsf_json_enabled", "maybe") + .expect_err("invalid bool should fail"); assert!(err.to_string().contains("invalid bool value")); } diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index bf35811643..0ff6d06d6c 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -33,10 +33,6 @@ default = ["telemetry"] ## `--no-default-features` (plus any other features you need) for a build that ## contains no telemetry endpoint, no HTTP client, and no emission code at all. telemetry = ["dep:reqwest", "dep:chrono"] -## Include test-only settings (dummy_bool, dummy_int) in the registry. -## Off by default so production builds have an empty registry. -## Enabled by e2e tests and during development. -dev-settings = [] ## Expose proposals::test_helpers (`ProposalsFlagGuard`) to downstream test ## code in other crates. Enabled by openshell-sandbox and ## openshell-supervisor-network dev builds. diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index 3ff1f36f8e..156e4c3845 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -137,20 +137,6 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ kind: SettingValueKind::String, allowed_string_values: Some(PROPOSAL_APPROVAL_MODE_VALUES), }, - // Test-only keys live behind the `dev-settings` feature flag so they - // don't appear in production builds. - #[cfg(feature = "dev-settings")] - RegisteredSetting { - key: "dummy_int", - kind: SettingValueKind::Int, - allowed_string_values: None, - }, - #[cfg(feature = "dev-settings")] - RegisteredSetting { - key: "dummy_bool", - kind: SettingValueKind::Bool, - allowed_string_values: None, - }, ]; /// Resolve a setting descriptor from the registry by key. @@ -187,15 +173,6 @@ mod tests { registered_keys_csv, setting_for_key, }; - #[cfg(feature = "dev-settings")] - #[test] - fn setting_for_key_returns_dev_entries() { - let setting = setting_for_key("dummy_bool").expect("dummy_bool should be registered"); - assert_eq!(setting.kind, SettingValueKind::Bool); - let setting = setting_for_key("dummy_int").expect("dummy_int should be registered"); - assert_eq!(setting.kind, SettingValueKind::Int); - } - #[test] fn setting_for_key_returns_none_for_unknown() { assert!(setting_for_key("nonexistent_key").is_none()); diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 39a26b14e7..b5c9b34d71 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -105,7 +105,6 @@ default = ["telemetry"] ## that contains no telemetry endpoint, HTTP client, or emission code. telemetry = ["openshell-core/telemetry"] bundled-z3 = ["openshell-prover/bundled-z3"] -dev-settings = ["openshell-core/dev-settings"] test-support = [] [dev-dependencies] diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 770bb71cca..b4fd093ea2 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -8895,24 +8895,22 @@ mod tests { assert!(err.message().contains("unknown setting key")); } - #[cfg(feature = "dev-settings")] #[test] fn proto_setting_to_stored_rejects_type_mismatch() { let value = SettingValue { value: Some(setting_value::Value::StringValue("true".to_string())), }; - let err = proto_setting_to_stored("dummy_bool", &value).unwrap_err(); + let err = proto_setting_to_stored("ocsf_json_enabled", &value).unwrap_err(); assert_eq!(err.code(), Code::InvalidArgument); assert!(err.message().contains("expects bool value")); } - #[cfg(feature = "dev-settings")] #[test] fn proto_setting_to_stored_accepts_bool_for_registered_bool_key() { let value = SettingValue { value: Some(setting_value::Value::BoolValue(true)), }; - let stored = proto_setting_to_stored("dummy_bool", &value).unwrap(); + let stored = proto_setting_to_stored("ocsf_json_enabled", &value).unwrap(); assert_eq!(stored, StoredSettingValue::Bool(true)); } @@ -8988,17 +8986,19 @@ mod tests { ); } - #[cfg(feature = "dev-settings")] #[test] fn merge_effective_settings_global_overrides_sandbox_key() { let global = StoredSettings { revision: 2, settings: [ ( - "log_level".to_string(), - StoredSettingValue::String("warn".to_string()), + settings::PROVIDERS_V2_ENABLED_KEY.to_string(), + StoredSettingValue::Bool(false), + ), + ( + settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), + StoredSettingValue::Bool(false), ), - ("dummy_int".to_string(), StoredSettingValue::Int(7)), ] .into_iter() .collect(), @@ -9008,10 +9008,13 @@ mod tests { revision: 1, settings: [ ( - "log_level".to_string(), - StoredSettingValue::String("debug".to_string()), + settings::PROVIDERS_V2_ENABLED_KEY.to_string(), + StoredSettingValue::Bool(true), + ), + ( + "ocsf_json_enabled".to_string(), + StoredSettingValue::Bool(true), ), - ("dummy_bool".to_string(), StoredSettingValue::Bool(true)), ] .into_iter() .collect(), @@ -9019,39 +9022,45 @@ mod tests { }; let merged = merge_effective_settings(&global, &sandbox).unwrap(); - let log_level = merged.get("log_level").expect("log_level present"); - assert_eq!(log_level.scope, SettingScope::Global as i32); + let providers_v2 = merged + .get(settings::PROVIDERS_V2_ENABLED_KEY) + .expect("providers_v2_enabled present"); + assert_eq!(providers_v2.scope, SettingScope::Global as i32); assert_eq!( - log_level.value.as_ref().and_then(|v| v.value.as_ref()), - Some(&setting_value::Value::StringValue("warn".to_string())) + providers_v2.value.as_ref().and_then(|v| v.value.as_ref()), + Some(&setting_value::Value::BoolValue(false)) ); - let dummy_bool = merged.get("dummy_bool").expect("dummy_bool present"); - assert_eq!(dummy_bool.scope, SettingScope::Sandbox as i32); + let ocsf_json = merged + .get("ocsf_json_enabled") + .expect("ocsf_json_enabled present"); + assert_eq!(ocsf_json.scope, SettingScope::Sandbox as i32); - let dummy_int = merged.get("dummy_int").expect("dummy_int present"); - assert_eq!(dummy_int.scope, SettingScope::Global as i32); + let proposals = merged + .get(settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY) + .expect("agent_policy_proposals_enabled present"); + assert_eq!(proposals.scope, SettingScope::Global as i32); } - #[cfg(feature = "dev-settings")] #[test] fn merge_effective_settings_sandbox_scoped_value_has_sandbox_scope() { let global = StoredSettings::default(); let sandbox = StoredSettings { revision: 1, - settings: [( - "log_level".to_string(), - StoredSettingValue::String("debug".to_string()), - )] - .into_iter() + settings: std::iter::once(( + "ocsf_json_enabled".to_string(), + StoredSettingValue::Bool(true), + )) .collect(), ..Default::default() }; let merged = merge_effective_settings(&global, &sandbox).unwrap(); - let log_level = merged.get("log_level").expect("log_level present"); - assert_eq!(log_level.scope, SettingScope::Sandbox as i32); - assert!(log_level.value.is_some()); + let ocsf_json = merged + .get("ocsf_json_enabled") + .expect("ocsf_json_enabled present"); + assert_eq!(ocsf_json.scope, SettingScope::Sandbox as i32); + assert!(ocsf_json.value.is_some()); } #[test] @@ -9274,9 +9283,10 @@ mod tests { "log_level".to_string(), StoredSettingValue::String("error".to_string()), ); - settings - .settings - .insert("dummy_bool".to_string(), StoredSettingValue::Bool(true)); + settings.settings.insert( + "ocsf_json_enabled".to_string(), + StoredSettingValue::Bool(true), + ); settings.revision = 5; save_global_settings(&store, &settings).await.unwrap(); @@ -9287,7 +9297,7 @@ mod tests { Some(&StoredSettingValue::String("error".to_string())) ); assert_eq!( - loaded.settings.get("dummy_bool"), + loaded.settings.get("ocsf_json_enabled"), Some(&StoredSettingValue::Bool(true)) ); } @@ -9298,9 +9308,10 @@ mod tests { let sandbox_name = "my-sandbox"; let mut settings = StoredSettings::default(); - settings - .settings - .insert("dummy_int".to_string(), StoredSettingValue::Int(99)); + settings.settings.insert( + settings::PROPOSAL_APPROVAL_MODE_KEY.to_string(), + StoredSettingValue::String("auto".to_string()), + ); settings.revision = 3; save_sandbox_settings(&store, sandbox_name, &settings) .await @@ -9309,8 +9320,8 @@ mod tests { let loaded = load_sandbox_settings(&store, sandbox_name).await.unwrap(); assert_eq!(loaded.revision, 3); assert_eq!( - loaded.settings.get("dummy_int"), - Some(&StoredSettingValue::Int(99)) + loaded.settings.get(settings::PROPOSAL_APPROVAL_MODE_KEY), + Some(&StoredSettingValue::String("auto".to_string())) ); } @@ -9434,14 +9445,15 @@ mod tests { let store = test_store().await; let mut global = StoredSettings::default(); - global - .settings - .insert("dummy_int".to_string(), StoredSettingValue::Int(42)); + global.settings.insert( + "ocsf_json_enabled".to_string(), + StoredSettingValue::Bool(true), + ); global.revision = 1; save_global_settings(&store, &global).await.unwrap(); let loaded_global = load_global_settings(&store).await.unwrap(); - assert!(loaded_global.settings.contains_key("dummy_int")); + assert!(loaded_global.settings.contains_key("ocsf_json_enabled")); } #[tokio::test] diff --git a/e2e/rust/e2e-docker.sh b/e2e/rust/e2e-docker.sh index a020f87c83..70e9835bde 100755 --- a/e2e/rust/e2e-docker.sh +++ b/e2e/rust/e2e-docker.sh @@ -12,7 +12,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_DOCKER_TEST:-smoke}" E2E_FEATURES="${OPENSHELL_E2E_DOCKER_FEATURES:-e2e,e2e-docker}" -cargo build -p openshell-cli --features openshell-core/dev-settings +cargo build -p openshell-cli exec "${ROOT}/e2e/with-docker-gateway.sh" \ cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ diff --git a/e2e/rust/e2e-kubernetes.sh b/e2e/rust/e2e-kubernetes.sh index 0644a06183..ec0faefed5 100755 --- a/e2e/rust/e2e-kubernetes.sh +++ b/e2e/rust/e2e-kubernetes.sh @@ -21,7 +21,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_FEATURES="${OPENSHELL_E2E_KUBERNETES_FEATURES:-e2e,e2e-host-gateway,e2e-kubernetes}" -cargo build -p openshell-cli --features openshell-core/dev-settings +cargo build -p openshell-cli test_filter=() if [ -n "${OPENSHELL_E2E_KUBE_TEST:-}" ]; then diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index 5f325d0d27..26843e128c 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -12,7 +12,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" E2E_TEST="${OPENSHELL_E2E_PODMAN_TEST:-}" E2E_FEATURES="${OPENSHELL_E2E_PODMAN_FEATURES:-e2e-podman}" -cargo build -p openshell-cli --features openshell-core/dev-settings +cargo build -p openshell-cli TEST_ARGS=( cargo test --manifest-path "${ROOT}/e2e/rust/Cargo.toml" diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 926821e539..467d419f02 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -91,8 +91,7 @@ echo "==> Building openshell-gateway, openshell-driver-vm, openshell (CLI)" cargo build \ -p openshell-server \ -p openshell-driver-vm \ - -p openshell-cli \ - --features openshell-core/dev-settings + -p openshell-cli if [ "$(uname -s)" = "Darwin" ]; then echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" diff --git a/e2e/rust/tests/settings_management.rs b/e2e/rust/tests/settings_management.rs index 69cb7cf161..e69269d7d8 100644 --- a/e2e/rust/tests/settings_management.rs +++ b/e2e/rust/tests/settings_management.rs @@ -21,7 +21,7 @@ use openshell_e2e::harness::output::strip_ansi; use openshell_e2e::harness::sandbox::SandboxGuard; use tokio::time::{Instant, sleep}; -const TEST_KEY: &str = "dummy_bool"; +const TEST_KEY: &str = "ocsf_json_enabled"; static SETTINGS_E2E_LOCK: Mutex<()> = Mutex::new(()); struct CliResult { @@ -231,7 +231,7 @@ async fn settings_global_override_round_trip() { assert!( set_global .clean_output - .contains("Set global setting dummy_bool=false"), + .contains(&format!("Set global setting {TEST_KEY}=false")), "expected global set output:\n{}", set_global.clean_output ); @@ -289,7 +289,7 @@ async fn settings_global_override_round_trip() { assert!( delete_global .clean_output - .contains("Deleted global setting dummy_bool"), + .contains(&format!("Deleted global setting {TEST_KEY}")), "expected global delete confirmation in output:\n{}", delete_global.clean_output ); diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index 8da3d07064..a52b4c0770 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -186,13 +186,11 @@ e2e_build_gateway_binaries() { echo "Building openshell-gateway..." cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway \ - --features openshell-core/dev-settings + -p openshell-server --bin openshell-gateway echo "Building openshell-cli..." cargo build "${jobs[@]}" \ - -p openshell-cli --bin openshell \ - --features openshell-core/dev-settings + -p openshell-cli if [ ! -x "${target_dir}/debug/openshell-gateway" ]; then echo "ERROR: expected openshell-gateway binary at ${target_dir}/debug/openshell-gateway" >&2 diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index 99cc2cf4b8..8d1334b8c6 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -153,7 +153,7 @@ build_component_for_arch() { resolve_component "$component" target="$(target_triple "$arch" "$target_libc")" stage="${ROOT}/deploy/docker/.build/prebuilt-binaries/${arch}" - features="${EXTRA_CARGO_FEATURES:-openshell-core/dev-settings}" + features="${EXTRA_CARGO_FEATURES:-}" if [[ "$component" == "gateway" && " ${features} " != *" bundled-z3 "* ]]; then features="${features} bundled-z3" fi diff --git a/tasks/test.toml b/tasks/test.toml index 3ee4b6ab5a..444ea15e16 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -60,14 +60,14 @@ hide = true ["e2e:rust"] description = "Run Rust CLI e2e tests against a Docker-backed gateway" run = [ - "cargo build -p openshell-cli --features openshell-core/dev-settings", + "cargo build -p openshell-cli", "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker", ] ["e2e:websocket-conformance"] description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" run = [ - "cargo build -p openshell-cli --features openshell-core/dev-settings", + "cargo build -p openshell-cli", "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test websocket_conformance", ] @@ -110,7 +110,7 @@ run = "e2e/rust/e2e-docker.sh" ["e2e:mechanistic-smoke"] description = "Run mechanistic L4 smoke against a Docker-backed gateway" run = [ - "cargo build -p openshell-cli --features openshell-core/dev-settings", + "cargo build -p openshell-cli", "e2e/with-docker-gateway.sh bash -lc 'target/debug/openshell settings set --global --key agent_policy_proposals_enabled --value true --yes && OPENSHELL_BIN=$PWD/target/debug/openshell bash e2e/policy-advisor/mechanistic-smoke.sh'", ] From c636e70a75a4ed815a09d5618a29c5d72512f142 Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Thu, 25 Jun 2026 09:18:25 -0500 Subject: [PATCH 03/20] fix(e2e): make postgres fixture compatible with OpenShift (#2002) --- e2e/kubernetes/postgres-fixture.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/e2e/kubernetes/postgres-fixture.yaml b/e2e/kubernetes/postgres-fixture.yaml index 390145145b..2ab422fd0f 100644 --- a/e2e/kubernetes/postgres-fixture.yaml +++ b/e2e/kubernetes/postgres-fixture.yaml @@ -16,6 +16,7 @@ metadata: type: Opaque stringData: password: openshell-e2e-postgres + uri: postgresql://openshell:openshell-e2e-postgres@openshell-e2e-postgres.openshell.svc/openshell --- apiVersion: apps/v1 kind: Deployment @@ -48,6 +49,13 @@ spec: key: password - name: POSTGRES_DB value: openshell + # OpenShift assigns a random UID from the namespace range instead of + # running as the image's "postgres" user (UID 70). The postgres + # entrypoint fails to chmod /var/lib/postgresql/data because it is + # owned by UID 70. Setting PGDATA to a subdirectory causes postgres + # to mkdir+chmod a new directory it owns, which succeeds. + - name: PGDATA + value: /var/lib/postgresql/data/pgdata ports: - name: postgres containerPort: 5432 @@ -61,6 +69,16 @@ spec: periodSeconds: 3 timeoutSeconds: 2 failureThreshold: 20 + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + - name: postgres-run + mountPath: /var/run/postgresql + volumes: + - name: postgres-data + emptyDir: {} + - name: postgres-run + emptyDir: {} --- apiVersion: v1 kind: Service From d93293ad12c581836a3794172d68a8b1cfa5441d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 25 Jun 2026 16:51:27 +0200 Subject: [PATCH 04/20] fix(e2e): stabilize local Docker smoke test (#1935) * fix(docker): honor configured supervisor image Signed-off-by: Evan Lezar * fix(cli): isolate ssh from host linker environment Signed-off-by: Evan Lezar --------- Signed-off-by: Evan Lezar --- crates/openshell-driver-docker/README.md | 9 +- crates/openshell-driver-docker/src/lib.rs | 103 +++++++++++++------- crates/openshell-driver-docker/src/tests.rs | 30 ++++++ docs/reference/gateway-config.mdx | 1 + e2e/with-docker-gateway.sh | 101 ++++++------------- 5 files changed, 133 insertions(+), 111 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index aedfa4e449..4f2977adde 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -79,10 +79,11 @@ The Docker driver bind-mounts a host-side Linux `openshell-sandbox` binary into each sandbox container. Resolution order is: 1. `supervisor_bin` in `[openshell.drivers.docker]`. -2. A sibling `openshell-sandbox` next to the running `openshell-gateway` binary. -3. A local Linux cargo target build for the Docker daemon architecture. -4. `supervisor_image` in `[openshell.drivers.docker]`, or the - release-matched default supervisor image, extracting `/openshell-sandbox`. +2. `supervisor_image` in `[openshell.drivers.docker]`, extracting + `/openshell-sandbox` from that image. +3. A sibling `openshell-sandbox` next to the running `openshell-gateway` binary. +4. A local Linux cargo target build for the Docker daemon architecture. +5. The release-matched default supervisor image, extracting `/openshell-sandbox`. Release and Docker-image gateway builds bake the matching supervisor image tag into the binary at compile time. The default Docker supervisor image is not diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 6ced548921..913054934e 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -80,7 +80,8 @@ const DOCKER_NETWORK_DRIVER: &str = "bridge"; /// Default image holding the Linux `openshell-sandbox` binary. The gateway /// pulls this image and extracts the binary to a host-side cache when no -/// explicit `supervisor_bin` override or local build is available. +/// explicit `supervisor_bin`, configured `supervisor_image`, sibling binary, +/// or local build is available. const DEFAULT_DOCKER_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/supervisor"; /// Return the default `ghcr.io/nvidia/openshell/supervisor:` reference @@ -156,10 +157,9 @@ pub struct DockerComputeConfig { /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. pub supervisor_bin: Option, - /// Optional override for the image the gateway pulls to extract the - /// Linux `openshell-sandbox` binary when no explicit binary path or - /// local build is available. Defaults to - /// `ghcr.io/nvidia/openshell/supervisor:`. + /// Optional image used to extract the Linux `openshell-sandbox` binary. + /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for + /// the full resolution order. pub supervisor_image: Option, /// Host-side CA certificate for Docker sandbox mTLS. @@ -2978,56 +2978,89 @@ fn normalize_docker_arch(arch: &str) -> String { } } -pub(crate) async fn resolve_supervisor_bin( - docker: &Docker, +#[derive(Debug, Eq, PartialEq)] +enum SupervisorBinSource { + Binary(PathBuf), + Image(String), +} + +fn resolve_supervisor_bin_source( docker_config: &DockerComputeConfig, - daemon_arch: &str, -) -> CoreResult { + current_exe: Option<&Path>, + target_candidates: &[PathBuf], +) -> CoreResult { // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. if let Some(path) = docker_config.supervisor_bin.clone() { let path = canonicalize_existing_file(&path, "docker supervisor binary")?; validate_linux_elf_binary(&path)?; - return Ok(path); + return Ok(SupervisorBinSource::Binary(path)); + } + + // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. + // A configured image should be the source of truth even when a local + // developer build is present under target/. + if let Some(image) = docker_config.supervisor_image.clone() { + return Ok(SupervisorBinSource::Image(image)); } - // Tier 2: sibling `openshell-sandbox` next to the running gateway + // Tier 3: sibling `openshell-sandbox` next to the running gateway // (release artifact layout). Linux-only because the sibling must be a // Linux ELF to bind-mount into a Linux container. - if cfg!(target_os = "linux") { - let current_exe = std::env::current_exe() - .map_err(|err| Error::config(format!("failed to resolve current executable: {err}")))?; - if let Some(parent) = current_exe.parent() { - let sibling = parent.join("openshell-sandbox"); - if sibling.is_file() { - let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(path); - } + if cfg!(target_os = "linux") + && let Some(current_exe) = current_exe + && let Some(parent) = current_exe.parent() + { + let sibling = parent.join("openshell-sandbox"); + if sibling.is_file() { + let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; + if validate_linux_elf_binary(&path).is_ok() { + return Ok(SupervisorBinSource::Binary(path)); } } } - // Tier 3: local cargo target build (developer workflow). Preferred - // over a registry pull when available because it matches whatever the - // developer just built. - let target_candidates = linux_supervisor_candidates(daemon_arch); - for candidate in &target_candidates { + // Tier 4: local cargo target build (developer workflow). Preferred + // over the default registry image when available because it matches + // whatever the developer just built. + for candidate in target_candidates { if candidate.is_file() { let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; if validate_linux_elf_binary(&path).is_ok() { - return Ok(path); + return Ok(SupervisorBinSource::Binary(path)); } } } - // Tier 4: pull the supervisor image from a registry and extract the - // binary to a host-side cache keyed by image content digest. This is - // the default path for released gateway binaries. - let image = docker_config - .supervisor_image - .clone() - .unwrap_or_else(default_docker_supervisor_image); - extract_supervisor_bin_from_image(docker, &image).await + // Tier 5: pull the release-matched default supervisor image and extract + // the binary to a host-side cache keyed by image content digest. + Ok(SupervisorBinSource::Image(default_docker_supervisor_image())) +} + +pub(crate) async fn resolve_supervisor_bin( + docker: &Docker, + docker_config: &DockerComputeConfig, + daemon_arch: &str, +) -> CoreResult { + let current_exe = + if cfg!(target_os = "linux") + && docker_config.supervisor_bin.is_none() + && docker_config.supervisor_image.is_none() + { + Some(std::env::current_exe().map_err(|err| { + Error::config(format!("failed to resolve current executable: {err}")) + })?) + } else { + None + }; + let target_candidates = linux_supervisor_candidates(daemon_arch); + + match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? + { + SupervisorBinSource::Binary(path) => Ok(path), + SupervisorBinSource::Image(image) => { + extract_supervisor_bin_from_image(docker, &image).await + } + } } fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 4f5b266177..923c6d618d 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -1865,6 +1865,36 @@ fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { ); } +#[test] +fn configured_supervisor_image_takes_precedence_over_local_binaries() { + let tempdir = TempDir::new().unwrap(); + let bin_dir = tempdir.path().join("bin"); + fs::create_dir_all(&bin_dir).unwrap(); + let current_exe = bin_dir.join("openshell-gateway"); + let sibling = bin_dir.join("openshell-sandbox"); + fs::write(¤t_exe, b"gateway").unwrap(); + fs::write(&sibling, b"\x7fELFsibling").unwrap(); + + let local_build = tempdir.path().join("target/openshell-sandbox"); + fs::create_dir_all(local_build.parent().unwrap()).unwrap(); + fs::write(&local_build, b"\x7fELFlocal").unwrap(); + + let source = resolve_supervisor_bin_source( + &DockerComputeConfig { + supervisor_image: Some("example.com/openshell/supervisor:test".to_string()), + ..Default::default() + }, + Some(¤t_exe), + &[local_build], + ) + .unwrap(); + + assert_eq!( + source, + SupervisorBinSource::Image("example.com/openshell/supervisor:test".to_string()) + ); +} + #[test] fn docker_supervisor_image_tag_prefers_explicit_build_tags() { assert_eq!( diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index ff45421368..d820a131dc 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -218,6 +218,7 @@ sandbox_namespace = "docker-dev" grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" +# When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 5dc61e6c1b..fe9156f980 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -64,6 +64,9 @@ require_container_engine_lane() { } require_container_engine_lane docker Docker +CONTAINER_ENGINE_QUIET="${CONTAINER_ENGINE_QUIET:-1}" +# shellcheck source=tasks/scripts/container-engine.sh +source "${ROOT}/tasks/scripts/container-engine.sh" github_actions_host_docker_tmpdir() { if [ "${GITHUB_ACTIONS:-}" != "true" ] \ @@ -111,7 +114,6 @@ DOCKER_NETWORK_NAME="" DOCKER_NETWORK_CONNECTED_CONTAINER="" DOCKER_NETWORK_MANAGED=0 GPU_MODE="${OPENSHELL_E2E_DOCKER_GPU:-0}" -DOCKER_SUPERVISOR_ARGS=() # Isolate CLI/SDK gateway metadata from the developer's real config. export XDG_CONFIG_HOME="${WORKDIR}/config" @@ -292,25 +294,6 @@ if [ "${GPU_MODE}" = "1" ]; then fi fi -normalize_arch() { - case "$1" in - x86_64|amd64) echo "amd64" ;; - aarch64|arm64) echo "arm64" ;; - *) echo "$1" ;; - esac -} - -linux_target_triple() { - case "$1" in - amd64) echo "x86_64-unknown-linux-gnu" ;; - arm64) echo "aarch64-unknown-linux-gnu" ;; - *) - echo "ERROR: unsupported Docker daemon architecture '$1'" >&2 - exit 2 - ;; - esac -} - resolve_docker_supervisor_image() { if [ -n "${OPENSHELL_DOCKER_SUPERVISOR_IMAGE:-}" ]; then printf '%s\n' "${OPENSHELL_DOCKER_SUPERVISOR_IMAGE}" @@ -333,7 +316,7 @@ resolve_docker_supervisor_image() { return 0 fi - printf '%s\n' "" + printf '%s\n' "openshell/supervisor:dev" } docker_pull_with_retry() { @@ -362,6 +345,27 @@ docker_pull_with_retry() { return 1 } +build_local_docker_supervisor_image_if_required() { + local image=$1 + + if [ "${image}" != "openshell/supervisor:dev" ]; then + return 0 + fi + + local daemon_arch + daemon_arch="$(ce_info_arch)" + + echo "Building local Docker supervisor image ${image} for linux/${daemon_arch}..." + CONTAINER_ENGINE=docker DOCKER_PLATFORM="linux/${daemon_arch}" IMAGE_TAG=dev \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor + if docker image inspect "${image}" >/dev/null 2>&1; then + return 0 + fi + + echo "ERROR: expected supervisor image '${image}' after local build." >&2 + exit 2 +} + ensure_docker_supervisor_image() { local image=$1 @@ -414,47 +418,12 @@ ensure_sandbox_image_available() { docker_pull_with_retry "${image}" } -DAEMON_ARCH="$(normalize_arch "$(docker info --format '{{.Architecture}}' 2>/dev/null || true)")" -SUPERVISOR_TARGET="$(linux_target_triple "${DAEMON_ARCH}")" -HOST_OS="$(uname -s)" -HOST_ARCH="$(normalize_arch "$(uname -m)")" -SUPERVISOR_OUT_DIR="${WORKDIR}/supervisor/${DAEMON_ARCH}" -SUPERVISOR_BIN="${SUPERVISOR_OUT_DIR}/openshell-sandbox" - -CARGO_BUILD_JOBS_ARG=() -if [ -n "${CARGO_BUILD_JOBS:-}" ]; then - CARGO_BUILD_JOBS_ARG=(-j "${CARGO_BUILD_JOBS}") -fi - e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN SUPERVISOR_IMAGE="$(resolve_docker_supervisor_image)" -if [ -n "${SUPERVISOR_IMAGE}" ]; then - ensure_docker_supervisor_image "${SUPERVISOR_IMAGE}" - echo "Using Docker supervisor image: ${SUPERVISOR_IMAGE}" - DOCKER_SUPERVISOR_ARGS=(--docker-supervisor-image "${SUPERVISOR_IMAGE}") -else - echo "Building openshell-sandbox for ${SUPERVISOR_TARGET}..." - mkdir -p "${SUPERVISOR_OUT_DIR}" - if [ "${HOST_OS}" = "Linux" ] && [ "${HOST_ARCH}" = "${DAEMON_ARCH}" ]; then - rustup target add "${SUPERVISOR_TARGET}" >/dev/null 2>&1 || true - cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - --release -p openshell-sandbox --target "${SUPERVISOR_TARGET}" - cp "${TARGET_DIR}/${SUPERVISOR_TARGET}/release/openshell-sandbox" "${SUPERVISOR_BIN}" - else - CONTAINER_ENGINE=docker \ - DOCKER_PLATFORM="linux/${DAEMON_ARCH}" \ - DOCKER_OUTPUT="type=local,dest=${SUPERVISOR_OUT_DIR}" \ - bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor-output - fi - - if [ ! -f "${SUPERVISOR_BIN}" ]; then - echo "ERROR: expected supervisor binary at ${SUPERVISOR_BIN}" >&2 - exit 1 - fi - chmod +x "${SUPERVISOR_BIN}" - DOCKER_SUPERVISOR_ARGS=(--docker-supervisor-bin "${SUPERVISOR_BIN}") -fi +build_local_docker_supervisor_image_if_required "${SUPERVISOR_IMAGE}" +ensure_docker_supervisor_image "${SUPERVISOR_IMAGE}" +echo "Using Docker supervisor image: ${SUPERVISOR_IMAGE}" DEFAULT_SANDBOX_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base:latest" SANDBOX_IMAGE="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE:-${OPENSHELL_SANDBOX_IMAGE:-${DEFAULT_SANDBOX_IMAGE}}}" @@ -521,19 +490,7 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" printf 'enable_bind_mounts = true\n' - # DOCKER_SUPERVISOR_ARGS holds either ("--docker-supervisor-bin" "") - # or ("--docker-supervisor-image" ""); both map to TOML keys on - # the docker driver config. - for ((i=0; i<${#DOCKER_SUPERVISOR_ARGS[@]}; i+=2)); do - case "${DOCKER_SUPERVISOR_ARGS[$i]}" in - --docker-supervisor-bin) - printf 'supervisor_bin = %s\n' "$(toml_string "${DOCKER_SUPERVISOR_ARGS[$((i+1))]}")" - ;; - --docker-supervisor-image) - printf 'supervisor_image = %s\n' "$(toml_string "${DOCKER_SUPERVISOR_ARGS[$((i+1))]}")" - ;; - esac - done + printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" fi From e4d7d41656718dae8e4f3606bd5fb278fd6bc3ab Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 25 Jun 2026 11:46:34 -0700 Subject: [PATCH 05/20] fix(snap): use snap-owned XDG directories (#1972) --- docs/about/installation.mdx | 9 +++++++-- snapcraft.yaml | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/about/installation.mdx b/docs/about/installation.mdx index 256015947e..87c8a83637 100644 --- a/docs/about/installation.mdx +++ b/docs/about/installation.mdx @@ -80,7 +80,7 @@ sudo loginctl enable-linger $USER Install the OpenShell snap from the Snap Store: ```shell -sudo snap install openshell --classic +sudo snap install openshell ``` The snap defines two apps: the `openshell` CLI and the `openshell.gateway` @@ -89,6 +89,11 @@ stores its database at `$SNAP_COMMON/gateway.db` (typically `/var/snap/openshell/common/gateway.db`). Create `$SNAP_COMMON/gateway.toml` when you need to override gateway settings. +The snap CLI stores per-user config, data, and state under `$SNAP_USER_COMMON`, +typically `~/snap/openshell/common`. Gateway registrations live under +`$SNAP_USER_COMMON/.config/openshell/gateways/` instead of +`~/.config/openshell/gateways/`. + ### Snap store installs When installing from the Snap Store, snapd automatically connects the `home`, @@ -108,7 +113,7 @@ manually. When installing a locally built `.snap` file, no plugs are connected by default: ```shell -sudo snap install ./openshell_*.snap --dangerous --classic +sudo snap install ./openshell_*.snap --dangerous sudo snap connect openshell:home sudo snap connect openshell:network sudo snap connect openshell:network-bind diff --git a/snapcraft.yaml b/snapcraft.yaml index 01d90366e7..f7354aa017 100644 --- a/snapcraft.yaml +++ b/snapcraft.yaml @@ -62,6 +62,10 @@ platforms: apps: openshell: command: bin/openshell + environment: + XDG_CONFIG_HOME: "$SNAP_USER_COMMON/.config" + XDG_DATA_HOME: "$SNAP_USER_COMMON/.local/share" + XDG_STATE_HOME: "$SNAP_USER_COMMON/.local/state" plugs: - home - network @@ -70,6 +74,10 @@ apps: term: command: bin/openshell term desktop: meta/gui/term.desktop + environment: + XDG_CONFIG_HOME: "$SNAP_USER_COMMON/.config" + XDG_DATA_HOME: "$SNAP_USER_COMMON/.local/share" + XDG_STATE_HOME: "$SNAP_USER_COMMON/.local/state" plugs: - home - network From 3ace968b64472c0e6601a71d9feaa7dc47d4d2b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:42:32 -0700 Subject: [PATCH 06/20] chore(deps): bump azure/setup-helm from 5.0.0 to 5.0.1 (#1996) Bumps [azure/setup-helm](https://github.com/azure/setup-helm) from 5.0.0 to 5.0.1. - [Release notes](https://github.com/azure/setup-helm/releases) - [Changelog](https://github.com/Azure/setup-helm/blob/main/CHANGELOG.md) - [Commits](https://github.com/azure/setup-helm/compare/dda3372f752e03dde6b3237bc9431cdc2f7a02a2...9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310) --- updated-dependencies: - dependency-name: azure/setup-helm dependency-version: 5.0.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-canary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index aa6d2a1aaf..2f205fd59c 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -208,7 +208,7 @@ jobs: KIND_GATEWAY_NAME: kind steps: - name: Install Helm - uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0 + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - name: Create kind cluster uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0 From f2ecadf6574f652d979a8eb6b899deb7f113c5f7 Mon Sep 17 00:00:00 2001 From: Yuedong Wu Date: Fri, 26 Jun 2026 15:30:25 +0800 Subject: [PATCH 07/20] refactor(cli): replace sandbox_create positional args with SandboxCreateConfig struct (#1997) Signed-off-by: Yuedong Wu --- crates/openshell-cli/src/main.rs | 38 +- crates/openshell-cli/src/run.rs | 104 +++-- .../sandbox_create_lifecycle_integration.rs | 355 +++++------------- 3 files changed, 197 insertions(+), 300 deletions(-) diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index fbed00e1a2..1e72f0ba5c 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -2678,25 +2678,27 @@ async fn main() -> Result<()> { apply_auth(&mut tls, &ctx.name); Box::pin(run::sandbox_create( endpoint, - name.as_deref(), - from.as_deref(), &ctx.name, - &upload_specs, - keep, - gpu_requirements, - cpu.as_deref(), - memory.as_deref(), - driver_config_json.as_deref(), - editor, - &providers, - policy.as_deref(), - forward, - &command, - tty_override, - auto_providers_override, - &labels_map, - &env_map, - &approval_mode, + run::SandboxCreateConfig { + name: name.as_deref(), + from: from.as_deref(), + uploads: &upload_specs, + keep, + gpu_requirements, + cpu: cpu.as_deref(), + memory: memory.as_deref(), + driver_config_json: driver_config_json.as_deref(), + editor, + providers: &providers, + policy: policy.as_deref(), + forward, + command: &command, + tty_override, + auto_providers_override, + labels: labels_map, + environment: env_map, + approval_mode: &approval_mode, + }, &tls, )) .await?; diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 29468aa49f..fee2641d8e 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -77,7 +77,7 @@ pub use crate::ssh::{ sandbox_ssh_proxy_by_name, sandbox_sync_down, sandbox_sync_up, sandbox_sync_up_files, }; pub use openshell_core::forward::{ - find_forward_by_port, list_forwards, stop_forward, stop_forwards_for_sandbox, + ForwardSpec, find_forward_by_port, list_forwards, stop_forward, stop_forwards_for_sandbox, }; #[derive(Debug, PartialEq, Eq)] @@ -1574,10 +1574,7 @@ pub fn doctor_check() -> Result<()> { Err(miette::miette!("docker info failed: {}", stderr.trim())) } -fn sandbox_should_persist( - keep: bool, - forward: Option<&openshell_core::forward::ForwardSpec>, -) -> bool { +fn sandbox_should_persist(keep: bool, forward: Option<&ForwardSpec>) -> bool { keep || forward.is_some() } @@ -1745,31 +1742,86 @@ async fn finalize_sandbox_create_session( session_result } +/// Configuration for creating a sandbox via the CLI. +/// +/// Infrastructure parameters (`server`, `gateway_name`, `tls`) remain positional +/// on the function signature, following the `provider_refresh_config(server, input, tls)` +/// precedent. This struct captures sandbox-specific options. +#[derive(Debug)] +pub struct SandboxCreateConfig<'a> { + pub name: Option<&'a str>, + pub from: Option<&'a str>, + pub uploads: &'a [(String, Option, bool)], + pub keep: bool, + pub gpu_requirements: Option, + pub cpu: Option<&'a str>, + pub memory: Option<&'a str>, + pub driver_config_json: Option<&'a str>, + pub editor: Option, + pub providers: &'a [String], + pub policy: Option<&'a str>, + pub forward: Option, + pub command: &'a [String], + pub tty_override: Option, + pub auto_providers_override: Option, + pub labels: HashMap, + pub environment: HashMap, + pub approval_mode: &'a str, +} + +impl Default for SandboxCreateConfig<'_> { + fn default() -> Self { + Self { + name: None, + from: None, + uploads: &[], + keep: false, + gpu_requirements: None, + cpu: None, + memory: None, + driver_config_json: None, + editor: None, + providers: &[], + policy: None, + forward: None, + command: &[], + tty_override: None, + auto_providers_override: None, + labels: HashMap::new(), + environment: HashMap::new(), + approval_mode: "manual", + } + } +} + /// Create a sandbox with default settings. -#[allow(clippy::too_many_arguments, clippy::implicit_hasher)] // user-facing CLI command; default hasher is fine pub async fn sandbox_create( server: &str, - name: Option<&str>, - from: Option<&str>, gateway_name: &str, - uploads: &[(String, Option, bool)], - keep: bool, - gpu_requirements: Option, - cpu: Option<&str>, - memory: Option<&str>, - driver_config_json: Option<&str>, - editor: Option, - providers: &[String], - policy: Option<&str>, - forward: Option, - command: &[String], - tty_override: Option, - auto_providers_override: Option, - labels: &HashMap, - environment: &HashMap, - approval_mode: &str, + config: SandboxCreateConfig<'_>, tls: &TlsOptions, ) -> Result<()> { + let SandboxCreateConfig { + name, + from, + uploads, + keep, + gpu_requirements, + cpu, + memory, + driver_config_json, + editor, + providers, + policy, + forward, + command, + tty_override, + auto_providers_override, + labels, + environment, + approval_mode, + } = config; + if editor.is_some() && !command.is_empty() { return Err(miette::miette!( "--editor cannot be used with a trailing command; use `openshell sandbox connect --editor ...` after the sandbox is ready" @@ -1846,14 +1898,14 @@ pub async fn sandbox_create( let request = CreateSandboxRequest { spec: Some(SandboxSpec { resource_requirements, - environment: environment.clone(), + environment, policy, providers: configured_providers, template, ..SandboxSpec::default() }), name: name.unwrap_or_default().to_string(), - labels: labels.clone(), + labels, }; let response = match client.create_sandbox(request).await { diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index 207386b840..ec8bd53743 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -1099,6 +1099,19 @@ fn gpu_requirements(count: Option) -> GpuResourceRequirements { GpuResourceRequirements { count } } +/// Shared defaults for integration tests. Note: `keep` is `true` here (most +/// tests expect persistent sandboxes) while `SandboxCreateConfig::default()` +/// sets `keep: false` (the safe production default). Tests that exercise +/// ephemeral behavior must explicitly override with `keep: false`. +fn test_config() -> run::SandboxCreateConfig<'static> { + run::SandboxCreateConfig { + keep: true, + tty_override: Some(false), + auto_providers_override: Some(false), + ..Default::default() + } +} + #[tokio::test] async fn sandbox_create_keeps_command_sessions_by_default() { let server = run_server().await; @@ -1110,25 +1123,12 @@ async fn sandbox_create_keeps_command_sessions_by_default() { run::sandbox_create( &server.endpoint, - Some("default-command"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("default-command"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1153,25 +1153,14 @@ async fn sandbox_create_sends_cpu_and_memory_limits_only() { run::sandbox_create( &server.endpoint, - Some("resources"), - None, "openshell", - &[], - true, - None, - Some("500m"), - Some("2Gi"), - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("resources"), + cpu: Some("500m"), + memory: Some("2Gi"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1230,25 +1219,15 @@ async fn sandbox_create_sends_driver_config_json() { run::sandbox_create( &server.endpoint, - Some("driver-config"), - None, "openshell", - &[], - true, - None, - None, - None, - Some(r#"{"kubernetes":{"pod":{"priority_class_name":"batch-low"}}}"#), - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("driver-config"), + driver_config_json: Some( + r#"{"kubernetes":{"pod":{"priority_class_name":"batch-low"}}}"#, + ), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1303,25 +1282,13 @@ async fn sandbox_create_sends_gpu_default_request() { run::sandbox_create( &server.endpoint, - Some("gpu-default"), - None, "openshell", - &[], - true, - Some(gpu_requirements(None)), - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("gpu-default"), + gpu_requirements: Some(gpu_requirements(None)), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1349,25 +1316,13 @@ async fn sandbox_create_sends_gpu_count_request() { run::sandbox_create( &server.endpoint, - Some("gpu-two"), - None, "openshell", - &[], - true, - Some(gpu_requirements(Some(2))), - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("gpu-two"), + gpu_requirements: Some(gpu_requirements(Some(2))), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1396,25 +1351,13 @@ async fn sandbox_create_does_not_infer_command_providers_when_v2_enabled() { run::sandbox_create( &server.endpoint, - Some("v2-no-inferred-provider"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["claude".to_string(), "--version".to_string()], - Some(true), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("v2-no-inferred-provider"), + command: &["claude".into(), "--version".into()], + tty_override: Some(true), + ..test_config() + }, &tls, ) .await @@ -1454,25 +1397,12 @@ async fn sandbox_create_returns_vm_error_without_waiting_for_timeout() { let started_at = Instant::now(); let err = run::sandbox_create( &server.endpoint, - Some("vm-error"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("vm-error"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1508,25 +1438,12 @@ async fn sandbox_create_keeps_waiting_while_vm_progress_arrives() { run::sandbox_create( &server.endpoint, - Some("vm-slow-progress"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("vm-slow-progress"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1554,25 +1471,12 @@ async fn sandbox_create_times_out_when_only_logs_arrive() { let started_at = Instant::now(); let err = run::sandbox_create( &server.endpoint, - Some("vm-log-churn"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("vm-log-churn"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1596,25 +1500,13 @@ async fn sandbox_create_deletes_command_sessions_with_no_keep() { run::sandbox_create( &server.endpoint, - Some("ephemeral-command"), - None, "openshell", - &[], - false, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("ephemeral-command"), + keep: false, + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1642,25 +1534,13 @@ async fn sandbox_create_deletes_shell_sessions_with_no_keep() { run::sandbox_create( &server.endpoint, - Some("ephemeral-shell"), - None, "openshell", - &[], - false, - None, - None, - None, - None, - None, - &[], - None, - None, - &[], - Some(true), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("ephemeral-shell"), + keep: false, + tty_override: Some(true), + ..test_config() + }, &tls, ) .await @@ -1688,25 +1568,12 @@ async fn sandbox_create_keeps_sandbox_with_hidden_keep_flag() { run::sandbox_create( &server.endpoint, - Some("persistent-keep"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("persistent-keep"), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1734,25 +1601,14 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() { run::sandbox_create( &server.endpoint, - Some("persistent-forward"), - None, "openshell", - &[], - false, - None, - None, - None, - None, - None, - &[], - None, - Some(openshell_core::forward::ForwardSpec::new(forward_port)), - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &HashMap::new(), - "manual", + run::SandboxCreateConfig { + name: Some("persistent-forward"), + keep: false, + forward: Some(openshell_core::forward::ForwardSpec::new(forward_port)), + command: &["echo".into(), "OK".into()], + ..test_config() + }, &tls, ) .await @@ -1880,31 +1736,18 @@ async fn sandbox_create_sends_environment_variables() { let tls = test_tls(&server); install_fake_ssh(&fake_ssh_dir); - let mut env_map = HashMap::new(); - env_map.insert("FOO".to_string(), "bar".to_string()); - env_map.insert("BAZ".to_string(), "qux=with=equals".to_string()); - run::sandbox_create( &server.endpoint, - Some("env-test"), - None, "openshell", - &[], - true, - None, - None, - None, - None, - None, - &[], - None, - None, - &["echo".to_string(), "OK".to_string()], - Some(false), - Some(false), - &HashMap::new(), - &env_map, - "manual", + run::SandboxCreateConfig { + name: Some("env-test"), + command: &["echo".into(), "OK".into()], + environment: HashMap::from([ + ("FOO".into(), "bar".into()), + ("BAZ".into(), "qux=with=equals".into()), + ]), + ..test_config() + }, &tls, ) .await From 75a317ea4573426f812e5d1017e0e75594df1654 Mon Sep 17 00:00:00 2001 From: st-gr <38470677+st-gr@users.noreply.github.com> Date: Fri, 26 Jun 2026 03:32:20 -0700 Subject: [PATCH 08/20] feat(server): support out-of-tree compute drivers via --compute-driver-socket (#1703) * feat(server): support remote compute driver endpoints Add named remote compute driver endpoint support to the gateway. Remote drivers are selected by a non-reserved compute driver name and either a CLI/env socket endpoint or [openshell.drivers.].socket_path. The VM driver now enters ComputeRuntime through the same acquired remote endpoint path, while Docker, Podman, and Kubernetes retain their in-process drivers. Require --drivers/OPENSHELL_DRIVERS when pairing an ad-hoc socket endpoint so the socket does not imply a magic driver name, and keep reserved in-tree names unavailable for unmanaged socket endpoints. Co-authored-by: Evan Lezar Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> Signed-off-by: Evan Lezar * test(server): cover remote compute driver UDS lifecycle Signed-off-by: Evan Lezar --------- Signed-off-by: st-gr <38470677+st-gr@users.noreply.github.com> Signed-off-by: Evan Lezar Co-authored-by: Evan Lezar --- architecture/compute-runtimes.md | 4 +- crates/openshell-core/src/config.rs | 70 ++++- crates/openshell-server/src/cli.rs | 188 +++++++++++- crates/openshell-server/src/compute/mod.rs | 265 ++++++++++++++--- crates/openshell-server/src/compute/vm.rs | 13 +- crates/openshell-server/src/config_file.rs | 4 +- crates/openshell-server/src/lib.rs | 292 ++++++++++++++---- crates/openshell-server/src/test_support.rs | 311 ++++++++++++++++++++ docs/reference/gateway-config.mdx | 21 ++ docs/reference/sandbox-compute-drivers.mdx | 31 +- 10 files changed, 1082 insertions(+), 117 deletions(-) create mode 100644 crates/openshell-server/src/test_support.rs diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f5c795c52f..f122fda5d7 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -33,7 +33,8 @@ when a sandbox create request asks for GPU resources. | Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. | | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API, OCI image volumes, and CDI GPU devices when available. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | -| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Gateway spawns `openshell-driver-vm` as a subprocess over a private, state-local Unix socket. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | +| Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | Per-sandbox CPU and memory values currently enter the driver layer through template resource limits. Docker and Podman apply them as runtime limits. @@ -82,6 +83,7 @@ The supervisor must be available inside each sandbox workload: | Podman | Read-only OCI image volume containing the supervisor binary. | | Kubernetes | Sandbox pod image or pod template configuration. | | VM | Embedded in the guest rootfs bundle. | +| Extension | Defined by the out-of-tree driver. | Driver-controlled environment variables must override sandbox image or template values for sandbox ID, sandbox name, gateway endpoint, relay socket path, TLS diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index eaaf1e4a01..c66d326103 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -4,6 +4,7 @@ //! Configuration management for `OpenShell` components. use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; use std::fmt; #[cfg(unix)] use std::io::{Read, Write}; @@ -69,6 +70,27 @@ impl ComputeDriverKind { } } +/// Normalize a configured compute driver name. +/// +/// Built-in driver names and custom remote driver names share the same +/// selection namespace. The normalized value is lowercase ASCII and may contain +/// letters, digits, `-`, and `_`. +pub fn normalize_compute_driver_name(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err("compute driver name cannot be empty".to_string()); + } + if !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) + { + return Err(format!( + "invalid compute driver name '{value}'. use ASCII letters, digits, '-' or '_'" + )); + } + Ok(value.to_ascii_lowercase()) +} + impl fmt::Display for ComputeDriverKind { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) @@ -358,7 +380,14 @@ pub struct Config { /// The config shape allows multiple drivers so the gateway can evolve /// toward multi-backend routing. Current releases require exactly one /// configured driver. - pub compute_drivers: Vec, + pub compute_drivers: Vec, + + /// Operator-provided endpoints for named remote compute drivers. + /// + /// This is populated by CLI/env inputs such as `--compute-driver-socket`. + /// TOML-authored endpoints live under `[openshell.drivers.]` and are + /// resolved by the gateway config loader. + pub compute_driver_endpoints: BTreeMap, /// TTL for SSH session tokens, in seconds. 0 disables expiry. pub ssh_session_ttl_secs: u64, @@ -559,6 +588,7 @@ impl Config { gateway_jwt: None, database_url: String::new(), compute_drivers: vec![], + compute_driver_endpoints: BTreeMap::new(), ssh_session_ttl_secs: default_ssh_session_ttl_secs(), grpc_rate_limit_requests: None, grpc_rate_limit_window_secs: None, @@ -614,11 +644,27 @@ impl Config { /// Create a new configuration with the configured compute drivers. #[must_use] - pub fn with_compute_drivers(mut self, drivers: I) -> Self + pub fn with_compute_drivers(mut self, drivers: I) -> Self where - I: IntoIterator, + I: IntoIterator, + D: ToString, { - self.compute_drivers = drivers.into_iter().collect(); + self.compute_drivers = drivers + .into_iter() + .map(|driver| driver.to_string()) + .collect(); + self + } + + /// Register a Unix domain socket endpoint for a named remote driver. + #[must_use] + pub fn with_compute_driver_endpoint( + mut self, + name: impl Into, + socket: impl Into, + ) -> Self { + self.compute_driver_endpoints + .insert(name.into(), socket.into()); self } @@ -766,8 +812,8 @@ mod tests { use super::is_reachable_unix_socket; use super::{ ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayJwtConfig, detect_driver, - docker_host_unix_socket_path, is_unix_socket, podman_socket_candidates_from_env, - podman_socket_responds, + docker_host_unix_socket_path, is_unix_socket, normalize_compute_driver_name, + podman_socket_candidates_from_env, podman_socket_responds, }; #[cfg(unix)] use std::io::{Read as _, Write as _}; @@ -803,6 +849,18 @@ mod tests { assert!(err.contains("unsupported compute driver 'firecracker'")); } + #[test] + fn compute_driver_name_normalization_accepts_builtin_and_custom_names() { + assert_eq!(normalize_compute_driver_name(" VM ").unwrap(), "vm"); + assert_eq!( + normalize_compute_driver_name("Kyma_GPU-1").unwrap(), + "kyma_gpu-1" + ); + + let err = normalize_compute_driver_name("kyma/gpu").unwrap_err(); + assert!(err.contains("invalid compute driver name")); + } + #[test] fn config_defaults_to_loopback_bind_address() { let expected: SocketAddr = "127.0.0.1:17670".parse().expect("valid address"); diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index ce7734262b..ef43dd4052 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -109,7 +109,17 @@ struct RunArgs { value_delimiter = ',', value_parser = parse_compute_driver )] - drivers: Vec, + drivers: Vec, + + /// Path to a Unix domain socket served by a remote compute driver + /// implementing `compute_driver.proto`. + /// + /// When set, the socket is associated with the single driver name supplied + /// by `--drivers` or `OPENSHELL_DRIVERS`. Reserved built-in driver names + /// such as Docker, Podman, Kubernetes, and VM do not accept socket + /// endpoints. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + compute_driver_socket: Option, /// Disable TLS entirely — listen on plaintext HTTP. /// Use this when the gateway sits behind a reverse proxy or tunnel @@ -235,6 +245,7 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> { if let Some(file) = file.as_ref() { merge_file_into_args(&mut args, &file.openshell.gateway, &matches); } + normalize_compute_driver_socket_args(&mut args, &matches)?; let local_tls = apply_runtime_defaults(&mut args)?; let local_jwt = defaults::complete_local_jwt_config()?; @@ -371,6 +382,13 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> { args.grpc_rate_limit_requests, args.grpc_rate_limit_window_seconds, )?; + if let Some(socket) = args.compute_driver_socket.clone() { + let driver = args + .drivers + .first() + .expect("normalize_compute_driver_socket_args sets a driver for socket endpoints"); + config = config.with_compute_driver_endpoint(driver.clone(), socket); + } if let Some(ttl) = file .as_ref() @@ -457,8 +475,8 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> { .into_diagnostic() } -fn parse_compute_driver(value: &str) -> std::result::Result { - value.parse() +fn parse_compute_driver(value: &str) -> std::result::Result { + openshell_core::config::normalize_compute_driver_name(value) } fn resolve_config_path(args: &RunArgs) -> Result> { @@ -657,10 +675,52 @@ fn validate_grpc_rate_limit_args(requests: Option, window_seconds: Option Result<()> { + let Some(socket) = args.compute_driver_socket.as_ref() else { + return Ok(()); + }; + if socket.as_os_str().is_empty() { + return Err(miette::miette!( + "--compute-driver-socket must not be an empty path" + )); + } + if arg_defaulted(matches, "drivers") { + return Err(miette::miette!( + "--compute-driver-socket requires --drivers or OPENSHELL_DRIVERS= to select a non-reserved compute driver name" + )); + } + + match args.drivers.as_slice() { + [driver] => { + let driver = openshell_core::config::normalize_compute_driver_name(driver) + .map_err(|err| miette::miette!("{err}"))?; + if matches!( + driver.parse::().ok(), + Some( + ComputeDriverKind::Docker + | ComputeDriverKind::Podman + | ComputeDriverKind::Kubernetes + | ComputeDriverKind::Vm + ) + ) { + return Err(miette::miette!( + "--compute-driver-socket cannot be combined with reserved built-in compute driver '{driver}'" + )); + } + args.drivers[0] = driver; + Ok(()) + } + drivers => Err(miette::miette!( + "--compute-driver-socket requires exactly one compute driver name, got: {}", + drivers.join(",") + )), + } +} + fn effective_single_driver(args: &RunArgs) -> Option { match args.drivers.as_slice() { [] => openshell_core::config::detect_driver(), - [driver] => Some(*driver), + [driver] => driver.parse().ok(), _ => None, } } @@ -1561,6 +1621,126 @@ ssh_session_ttl_secs = 1234 assert!(!super::is_singleplayer_driver(&multi)); } + #[test] + fn compute_driver_socket_flag_uses_explicit_driver_name() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); + let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--drivers", + "Kyma", + "--compute-driver-socket", + "/run/openshell/kyma.sock", + ]); + super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + assert_eq!( + args.compute_driver_socket.as_deref(), + Some(std::path::Path::new("/run/openshell/kyma.sock")) + ); + assert_eq!(args.drivers, ["kyma"]); + assert!(super::effective_single_driver(&args).is_none()); + } + + #[test] + fn compute_driver_socket_requires_explicit_driver_name() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); + let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--compute-driver-socket", + "/run/openshell/kyma.sock", + ]); + let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); + + assert!( + err.to_string().contains("requires --drivers "), + "unexpected error: {err}" + ); + } + + #[test] + fn compute_driver_socket_rejects_reserved_builtin_drivers() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); + let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--drivers", + "docker", + "--compute-driver-socket", + "/run/openshell/extension.sock", + ]); + let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); + assert!( + err.to_string() + .contains("cannot be combined with reserved built-in compute driver 'docker'"), + "unexpected error: {err}" + ); + } + + #[test] + fn compute_driver_socket_rejects_vm_endpoint() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::remove("OPENSHELL_COMPUTE_DRIVER_SOCKET"); + let _g2 = EnvVarGuard::remove("OPENSHELL_DRIVERS"); + + let (mut args, matches) = parse_with_args(&[ + "openshell-gateway", + "--db-url", + "sqlite::memory:", + "--drivers", + "vm", + "--compute-driver-socket", + "/run/openshell/vm.sock", + ]); + let err = super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap_err(); + assert!( + err.to_string() + .contains("cannot be combined with reserved built-in compute driver 'vm'"), + "unexpected error: {err}" + ); + } + + #[test] + fn compute_driver_socket_reads_from_env_var() { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _g1 = EnvVarGuard::set( + "OPENSHELL_COMPUTE_DRIVER_SOCKET", + "/var/run/openshell/kyma.sock", + ); + let _g2 = EnvVarGuard::set("OPENSHELL_DRIVERS", "kyma"); + + let (mut args, matches) = + parse_with_args(&["openshell-gateway", "--db-url", "sqlite::memory:"]); + super::normalize_compute_driver_socket_args(&mut args, &matches).unwrap(); + assert_eq!( + args.compute_driver_socket.as_deref(), + Some(std::path::Path::new("/var/run/openshell/kyma.sock")) + ); + assert_eq!(args.drivers, ["kyma"]); + } + #[test] fn file_populates_service_routing_fields() { let _lock = ENV_LOCK diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index e2fa1f8bc5..10953f1bf6 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -16,6 +16,7 @@ use crate::sandbox_watch::SandboxWatchBus; use crate::supervisor_session::SupervisorSessionRegistry; use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; +use hyper_util::rt::TokioIo; use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, @@ -40,12 +41,16 @@ use openshell_driver_podman::{ use prost::Message; use std::fmt; use std::net::SocketAddr; +use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +#[cfg(unix)] +use tokio::net::UnixStream; use tokio::sync::{Mutex, watch}; -use tonic::transport::Channel; +use tonic::transport::{Channel, Endpoint}; use tonic::{Code, Request, Status}; +use tower::service_fn; use tracing::{debug, info, warn}; type DriverWatchStream = Pin> + Send>>; @@ -105,11 +110,11 @@ pub use openshell_core::ComputeDriverError as ComputeError; #[derive(Debug)] pub struct ManagedDriverProcess { child: std::sync::Mutex>, - socket_path: std::path::PathBuf, + socket_path: PathBuf, } impl ManagedDriverProcess { - pub(crate) fn new(child: tokio::process::Child, socket_path: std::path::PathBuf) -> Self { + pub(crate) fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { Self { child: std::sync::Mutex::new(Some(child)), socket_path, @@ -126,6 +131,35 @@ impl Drop for ManagedDriverProcess { } } +#[derive(Debug)] +pub struct AcquiredRemoteDriverEndpoint { + pub(crate) name: String, + pub(crate) channel: Channel, + pub(crate) driver_process: Option>, +} + +impl AcquiredRemoteDriverEndpoint { + pub(crate) fn managed_builtin( + driver_kind: ComputeDriverKind, + channel: Channel, + driver_process: Arc, + ) -> Self { + Self { + name: driver_kind.as_str().to_string(), + channel, + driver_process: Some(driver_process), + } + } + + pub(crate) fn unmanaged(name: impl Into, channel: Channel) -> Self { + Self { + name: name.into(), + channel, + driver_process: None, + } + } +} + #[derive(Debug, Clone)] struct RemoteComputeDriver { channel: Channel, @@ -224,7 +258,7 @@ impl ComputeDriver for RemoteComputeDriver { #[derive(Clone)] pub struct ComputeRuntime { driver: SharedComputeDriver, - driver_kind: Option, + driver_name: String, shutdown_cleanup: Option>, startup_resume: Option>, _driver_process: Option>, @@ -248,7 +282,7 @@ impl fmt::Debug for ComputeRuntime { impl ComputeRuntime { #[allow(clippy::too_many_arguments)] async fn from_driver( - driver_kind: ComputeDriverKind, + driver_name: String, driver: SharedComputeDriver, shutdown_cleanup: Option>, startup_resume: Option>, @@ -261,15 +295,22 @@ impl ComputeRuntime { _allows_loopback_endpoints: bool, gateway_bind_addresses: Vec, ) -> Result { - let default_image = driver + let capabilities = driver .get_capabilities(Request::new(GetCapabilitiesRequest {})) .await .map_err(compute_error_from_status)? - .into_inner() - .default_image; + .into_inner(); + let driver_kind = driver_name.parse::().ok(); + info!( + configured_driver = %driver_name, + advertised_driver = %capabilities.driver_name, + in_tree = driver_kind.is_some(), + "Compute driver connected" + ); + let default_image = capabilities.default_image; Ok(Self { driver, - driver_kind: Some(driver_kind), + driver_name, shutdown_cleanup, startup_resume, _driver_process: driver_process, @@ -314,7 +355,7 @@ impl ComputeRuntime { let startup_resume: Arc = driver.clone(); let driver: SharedComputeDriver = driver; Self::from_driver( - ComputeDriverKind::Docker, + ComputeDriverKind::Docker.as_str().to_string(), driver, Some(shutdown_cleanup), Some(startup_resume), @@ -343,7 +384,7 @@ impl ComputeRuntime { .map_err(|err| ComputeError::Message(err.to_string()))?; let driver: SharedComputeDriver = Arc::new(ComputeDriverService::new(driver)); Self::from_driver( - ComputeDriverKind::Kubernetes, + ComputeDriverKind::Kubernetes.as_str().to_string(), driver, None, None, @@ -359,22 +400,21 @@ impl ComputeRuntime { .await } - pub(crate) async fn new_remote_vm( - channel: Channel, - driver_process: Option>, + pub(crate) async fn new_remote_driver( + endpoint: AcquiredRemoteDriverEndpoint, store: Arc, sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, ) -> Result { - let driver: SharedComputeDriver = Arc::new(RemoteComputeDriver::new(channel)); + let driver: SharedComputeDriver = Arc::new(RemoteComputeDriver::new(endpoint.channel)); Self::from_driver( - ComputeDriverKind::Vm, + endpoint.name, driver, None, None, - driver_process, + endpoint.driver_process, store, sandbox_index, sandbox_watch_bus, @@ -399,7 +439,7 @@ impl ComputeRuntime { .map_err(|err| ComputeError::Message(err.to_string()))?; let driver: SharedComputeDriver = Arc::new(PodmanDriverService::new(driver)); Self::from_driver( - ComputeDriverKind::Podman, + ComputeDriverKind::Podman.as_str().to_string(), driver, None, None, @@ -422,7 +462,7 @@ impl ComputeRuntime { #[must_use] pub fn driver_kind(&self) -> Option { - self.driver_kind + self.driver_name.parse().ok() } #[must_use] @@ -432,7 +472,7 @@ impl ComputeRuntime { pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), Status> { let driver_sandbox = - driver_sandbox_from_public(sandbox, self.driver_kind).map_err(|status| *status)?; + driver_sandbox_from_public(sandbox, &self.driver_name).map_err(|status| *status)?; self.driver .validate_sandbox_create(Request::new(ValidateSandboxCreateRequest { sandbox: Some(driver_sandbox), @@ -448,7 +488,7 @@ impl ComputeRuntime { ) -> Result { let sandbox_id = sandbox.object_id().to_string(); let mut driver_sandbox = - driver_sandbox_from_public(&sandbox, self.driver_kind).map_err(|status| *status)?; + driver_sandbox_from_public(&sandbox, &self.driver_name).map_err(|status| *status)?; // Create with MustCreate condition to prevent duplicate creation race self.sandbox_index.update_from_sandbox(&sandbox); @@ -1472,9 +1512,48 @@ impl ComputeRuntime { } } +/// Connect to an unmanaged remote compute driver that is already listening on +/// `socket_path` and return the acquired endpoint. +/// +/// The gateway does not spawn or own the driver process — the operator is +/// responsible for placing the driver alongside the gateway and granting the +/// gateway uid read/write on the socket. The host portion of the URL is +/// ignored because the connector resolves to the UDS rather than DNS. +#[cfg(unix)] +pub async fn connect_remote_compute_driver( + name: impl Into, + socket_path: &Path, +) -> Result { + let socket_path: PathBuf = socket_path.to_path_buf(); + let display_path = socket_path.clone(); + let channel = Endpoint::from_static("http://[::]:50051") + .connect_with_connector(service_fn(move |_: tonic::transport::Uri| { + let socket_path = socket_path.clone(); + async move { UnixStream::connect(socket_path).await.map(TokioIo::new) } + })) + .await + .map_err(|e| { + ComputeError::Message(format!( + "failed to connect to remote compute driver socket '{}': {e}", + display_path.display() + )) + })?; + Ok(AcquiredRemoteDriverEndpoint::unmanaged(name, channel)) +} + +#[cfg(not(unix))] +pub async fn connect_remote_compute_driver( + _name: impl Into, + _socket_path: &Path, +) -> Result { + Err(ComputeError::Message( + "remote compute driver endpoints require unix domain socket support".to_string(), + )) +} + fn driver_sandbox_from_public( sandbox: &Sandbox, - driver_kind: Option, + driver_name: &str, ) -> Result> { Ok(DriverSandbox { id: sandbox.object_id().to_string(), @@ -1483,7 +1562,7 @@ fn driver_sandbox_from_public( spec: sandbox .spec .as_ref() - .map(|spec| driver_sandbox_spec_from_public(spec, driver_kind)) + .map(|spec| driver_sandbox_spec_from_public(spec, driver_name)) .transpose()?, status: sandbox.status.as_ref().map(driver_status_from_public), }) @@ -1491,7 +1570,7 @@ fn driver_sandbox_from_public( fn driver_sandbox_spec_from_public( spec: &SandboxSpec, - driver_kind: Option, + driver_name: &str, ) -> Result> { Ok(DriverSandboxSpec { log_level: spec.log_level.clone(), @@ -1499,7 +1578,7 @@ fn driver_sandbox_spec_from_public( template: spec .template .as_ref() - .map(|template| driver_sandbox_template_from_public(template, driver_kind)) + .map(|template| driver_sandbox_template_from_public(template, driver_name)) .transpose()?, resource_requirements: spec.resource_requirements.as_ref().map(|requirements| { DriverSandboxResourceRequirements { @@ -1515,7 +1594,7 @@ fn driver_sandbox_spec_from_public( fn driver_sandbox_template_from_public( template: &SandboxTemplate, - driver_kind: Option, + driver_name: &str, ) -> Result> { Ok(DriverSandboxTemplate { image: template.image.clone(), @@ -1524,21 +1603,17 @@ fn driver_sandbox_template_from_public( environment: template.environment.clone(), resources: extract_typed_resources(&template.resources), platform_config: build_platform_config(template), - driver_config: select_driver_config(&template.driver_config, driver_kind)?, + driver_config: select_driver_config(&template.driver_config, driver_name)?, }) } fn select_driver_config( config: &Option, - driver_kind: Option, + driver_name: &str, ) -> Result, Box> { let Some(config) = config else { return Ok(None); }; - let Some(driver_kind) = driver_kind else { - return Ok(None); - }; - let driver_name = driver_kind.as_str(); let Some(value) = config.fields.get(driver_name) else { return Ok(None); }; @@ -2035,7 +2110,7 @@ impl ComputeDriver for NoopTestDriver { pub async fn new_test_runtime(store: Arc) -> ComputeRuntime { ComputeRuntime { driver: Arc::new(NoopTestDriver), - driver_kind: None, + driver_name: "test".to_string(), shutdown_cleanup: None, startup_resume: None, _driver_process: None, @@ -2125,8 +2200,7 @@ mod tests { .collect(), }; - let selected = - select_driver_config(&Some(config), Some(ComputeDriverKind::Kubernetes)).unwrap(); + let selected = select_driver_config(&Some(config), "kubernetes").unwrap(); let selected = selected.expect("kubernetes config should be selected"); assert!(selected.fields.contains_key("node")); @@ -2143,12 +2217,27 @@ mod tests { .collect(), }; - let selected = - select_driver_config(&Some(config), Some(ComputeDriverKind::Kubernetes)).unwrap(); + let selected = select_driver_config(&Some(config), "kubernetes").unwrap(); assert!(selected.is_none()); } + #[test] + fn select_driver_config_forwards_named_remote_driver_block() { + let config = prost_types::Struct { + fields: std::iter::once(( + "kyma".to_string(), + struct_value([("pool", string_value("gpu"))]), + )) + .collect(), + }; + + let selected = select_driver_config(&Some(config), "kyma").unwrap(); + let selected = selected.expect("named remote config should be selected"); + + assert!(selected.fields.contains_key("pool")); + } + #[test] fn select_driver_config_rejects_non_object_matching_driver_block() { let config = prost_types::Struct { @@ -2156,8 +2245,7 @@ mod tests { .collect(), }; - let err = - select_driver_config(&Some(config), Some(ComputeDriverKind::Kubernetes)).unwrap_err(); + let err = select_driver_config(&Some(config), "kubernetes").unwrap_err(); assert_eq!(err.code(), Code::InvalidArgument); assert!(err.message().contains("template.driver_config.kubernetes")); @@ -2277,7 +2365,7 @@ mod tests { let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); ComputeRuntime { driver, - driver_kind: None, + driver_name: "test-driver".to_string(), shutdown_cleanup: None, startup_resume, _driver_process: None, @@ -3363,6 +3451,103 @@ mod tests { ); } + #[tokio::test] + #[cfg(unix)] + async fn remote_compute_driver_forwards_lifecycle_calls_over_uds() { + use crate::test_support::{FakeComputeDriver, FakeComputeDriverCall}; + + let dir = tempfile::tempdir().unwrap(); + let socket_path = dir.path().join("compute-driver.sock"); + let driver = FakeComputeDriver::new() + .with_driver_name("fake-remote-driver") + .with_default_image("openshell/sandbox:remote"); + let _server = driver.serve_uds(&socket_path).unwrap(); + + let endpoint = connect_remote_compute_driver("external-test", &socket_path) + .await + .unwrap(); + let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + store, + SandboxIndex::new(), + SandboxWatchBus::new(), + TracingLogBus::new(), + Arc::new(SupervisorSessionRegistry::new()), + ) + .await + .unwrap(); + + let mut sandbox = sandbox_record("sb-uds", "uds-sandbox", SandboxPhase::Provisioning); + sandbox.spec = Some(SandboxSpec { + log_level: "debug".to_string(), + template: Some(SandboxTemplate { + image: "ghcr.io/nvidia/openshell/sandbox:test".to_string(), + driver_config: Some(prost_types::Struct { + fields: [ + ( + "external-test".to_string(), + struct_value([("pool", string_value("ci"))]), + ), + ( + "docker".to_string(), + struct_value([("network_mode", string_value("bridge"))]), + ), + ] + .into_iter() + .collect(), + }), + ..Default::default() + }), + ..Default::default() + }); + + runtime.validate_sandbox_create(&sandbox).await.unwrap(); + runtime.create_sandbox(sandbox, None).await.unwrap(); + assert!(runtime.delete_sandbox("uds-sandbox").await.unwrap()); + + let calls = driver.calls(); + assert_eq!(calls.len(), 4, "unexpected calls: {calls:?}"); + assert!(matches!(calls[0], FakeComputeDriverCall::GetCapabilities)); + + let validated = match &calls[1] { + FakeComputeDriverCall::ValidateSandboxCreate { + sandbox: Some(sandbox), + } => sandbox, + other => panic!("expected ValidateSandboxCreate call, got {other:?}"), + }; + assert_eq!(validated.id, "sb-uds"); + assert_eq!(validated.name, "uds-sandbox"); + let driver_config = validated + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .and_then(|template| template.driver_config.as_ref()) + .expect("selected driver_config should be forwarded"); + assert!(driver_config.fields.contains_key("pool")); + assert!(!driver_config.fields.contains_key("network_mode")); + + let created = match &calls[2] { + FakeComputeDriverCall::CreateSandbox { + sandbox: Some(sandbox), + } => sandbox, + other => panic!("expected CreateSandbox call, got {other:?}"), + }; + assert_eq!(created.id, "sb-uds"); + assert_eq!(created.name, "uds-sandbox"); + + match &calls[3] { + FakeComputeDriverCall::DeleteSandbox { + sandbox_id, + sandbox_name, + } => { + assert_eq!(sandbox_id, "sb-uds"); + assert_eq!(sandbox_name, "uds-sandbox"); + } + other => panic!("expected DeleteSandbox call, got {other:?}"), + } + } + #[tokio::test] async fn create_sandbox_returns_resource_version_one() { let runtime = test_runtime(Arc::new(TestDriver::default())).await; diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index efdc9daab4..be88047f33 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -29,6 +29,7 @@ //! trait implementation registering the VM driver against the generic //! interface. +use super::AcquiredRemoteDriverEndpoint; #[cfg(unix)] use super::ManagedDriverProcess; #[cfg(unix)] @@ -37,7 +38,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{Config, Error, Result}; +use openshell_core::{ComputeDriverKind, Config, Error, Result}; #[cfg(unix)] use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; #[cfg(unix)] @@ -451,7 +452,7 @@ pub fn compute_driver_guest_tls_paths( pub async fn spawn( config: &Config, vm_config: &VmComputeConfig, -) -> Result<(Channel, Arc)> { +) -> Result { if vm_config.grpc_endpoint.trim().is_empty() { return Err(Error::config( "grpc_endpoint is required when using the vm compute driver", @@ -507,14 +508,18 @@ pub async fn spawn( })?; let channel = wait_for_compute_driver(&socket_path, &mut child).await?; let process = Arc::new(ManagedDriverProcess::new(child, socket_path)); - Ok((channel, process)) + Ok(AcquiredRemoteDriverEndpoint::managed_builtin( + ComputeDriverKind::Vm, + channel, + process, + )) } #[cfg(not(unix))] pub async fn spawn( _config: &Config, _vm_config: &VmComputeConfig, -) -> Result<(Channel, std::sync::Arc)> { +) -> Result { Err(Error::config( "the vm compute driver requires unix domain socket support", )) diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 39cf02bbaa..3875756dc3 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -87,7 +87,7 @@ pub struct GatewayFileSection { // ── Drivers ────────────────────────────────────────────────────────── #[serde(default)] - pub compute_drivers: Option>, + pub compute_drivers: Option>, // ── Sandbox / SSH ──────────────────────────────────────────────────── #[serde(default)] @@ -631,7 +631,7 @@ version = 2 .expect("compute_drivers must be explicitly set in the RPM default config"); assert_eq!( drivers, - &[ComputeDriverKind::Podman], + &["podman".to_string()], "RPM default must pin compute_drivers to [podman] to prevent unexpected \ driver selection when Docker is also installed" ); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index dda8708e02..d0dbb16810 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -10,14 +10,18 @@ //! - mTLS support //! //! TODO(driver-abstraction): `build_compute_runtime` still switches on -//! [`ComputeDriverKind`] and calls driver-specific constructors -//! ([`ComputeRuntime::new_kubernetes`], [`compute::vm::spawn`] + -//! [`ComputeRuntime::new_remote_vm`]). Once we have a generalized compute -//! driver interface, the per-arm wiring here should collapse to a single -//! driver-agnostic path that asks each registered driver to produce a -//! [`Channel`](tonic::transport::Channel) and hands the rest of the gateway a -//! uniform [`ComputeRuntime`]. The remaining VM plumbing now lives in -//! [`compute::vm`]; keep this file driver-agnostic going forward. +//! built-in driver names and calls driver-specific constructors +//! ([`ComputeRuntime::new_kubernetes`], [`ComputeRuntime::new_docker`], +//! [`compute::vm::spawn`] + [`ComputeRuntime::new_remote_driver`], +//! [`ComputeRuntime::new_podman`]). Endpoint-backed drivers now share the +//! remote `compute_driver.proto` path, so new remote drivers should enter +//! through named endpoint acquisition rather than gateway-wide socket side +//! channels. Once we have a generalized compute-driver registry, the remaining +//! per-arm wiring here should collapse to driver construction records that +//! produce either an in-process `SharedComputeDriver` or an acquired remote +//! endpoint, then hand the rest of the gateway a uniform [`ComputeRuntime`]. +//! The VM launch plumbing now lives in [`compute::vm`]; keep this file limited +//! to selecting and acquiring drivers. mod auth; pub mod certgen; @@ -39,6 +43,8 @@ mod service_routing; mod ssh_sessions; pub mod supervisor_session; mod telemetry; +#[cfg(any(test, feature = "test-support"))] +pub mod test_support; mod tls; #[cfg(test)] pub(crate) mod tls_test_utils; @@ -47,6 +53,7 @@ mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; use openshell_core::{ComputeDriverKind, Config, Error, Result}; +use serde::Deserialize; use std::collections::HashMap; use std::io::ErrorKind; use std::net::SocketAddr; @@ -721,12 +728,14 @@ async fn build_compute_runtime( tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, ) -> Result { - let driver = configured_compute_driver(config)?; - info!(driver = %driver, "Using compute driver"); - warn_if_kubernetes_sandbox_jwt_expiry_disabled(config, driver); + let driver = configured_compute_driver(config, file)?; + info!(driver = %driver.name(), "Using compute driver"); + if let ConfiguredComputeDriver::Builtin(kind) = &driver { + warn_if_kubernetes_sandbox_jwt_expiry_disabled(config, *kind); + } match driver { - ComputeDriverKind::Kubernetes => { + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => { let mut k8s = kubernetes_config_from_file(file)?; if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { k8s.workspace_default_storage_size = size; @@ -742,7 +751,7 @@ async fn build_compute_runtime( .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) } - ComputeDriverKind::Docker => ComputeRuntime::new_docker( + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => ComputeRuntime::new_docker( config.clone(), docker_config.clone(), store, @@ -753,21 +762,7 @@ async fn build_compute_runtime( ) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))), - ComputeDriverKind::Vm => { - let (channel, driver_process) = compute::vm::spawn(config, vm_config).await?; - ComputeRuntime::new_remote_vm( - channel, - Some(driver_process), - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) - } - ComputeDriverKind::Podman => { + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => { let mut podman = podman_config_from_file(file)?; podman.gateway_port = config.bind_address.port(); if let Ok(p) = std::env::var("OPENSHELL_PODMAN_SOCKET") { @@ -789,6 +784,40 @@ async fn build_compute_runtime( .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) } + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => { + let endpoint = compute::vm::spawn(config, vm_config).await?; + ComputeRuntime::new_remote_driver( + endpoint, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + ) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + } + ConfiguredComputeDriver::Remote(remote) => { + let RemoteComputeDriverSelection { name, socket_path } = remote; + info!( + driver = %name, + socket = %socket_path.display(), + "Using remote compute driver endpoint" + ); + let endpoint = compute::connect_remote_compute_driver(name, &socket_path) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; + ComputeRuntime::new_remote_driver( + endpoint, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + ) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + } } } @@ -868,35 +897,117 @@ fn apply_podman_local_tls_defaults( Ok(()) } -fn configured_compute_driver(config: &Config) -> Result { +#[derive(Debug, Clone)] +enum ConfiguredComputeDriver { + Builtin(ComputeDriverKind), + Remote(RemoteComputeDriverSelection), +} + +impl ConfiguredComputeDriver { + fn name(&self) -> &str { + match self { + Self::Builtin(kind) => kind.as_str(), + Self::Remote(remote) => &remote.name, + } + } +} + +#[derive(Debug, Clone)] +struct RemoteComputeDriverSelection { + name: String, + socket_path: PathBuf, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct RemoteComputeDriverConfig { + socket_path: PathBuf, +} + +fn configured_compute_driver( + config: &Config, + file: Option<&config_file::ConfigFile>, +) -> Result { match config.compute_drivers.as_slice() { [] => match openshell_core::config::detect_driver() { Some(ComputeDriverKind::Vm) => Err(Error::config( "vm compute driver is opt-in only; set --drivers vm or OPENSHELL_DRIVERS=vm", )), - Some(driver) => Ok(driver), + Some(driver) => Ok(ConfiguredComputeDriver::Builtin(driver)), None => Err(Error::config( "no compute driver configured and auto-detection found no suitable driver; \ set --drivers or OPENSHELL_DRIVERS to kubernetes, podman, docker, or vm", )), }, - [ - driver @ (ComputeDriverKind::Kubernetes - | ComputeDriverKind::Vm - | ComputeDriverKind::Docker - | ComputeDriverKind::Podman), - ] => Ok(*driver), + [driver] => resolve_configured_compute_driver(driver, config, file), drivers => Err(Error::config(format!( "multiple compute drivers are not supported yet; configured drivers: {}", - drivers - .iter() - .map(ToString::to_string) - .collect::>() - .join(",") + drivers.join(",") ))), } } +fn resolve_configured_compute_driver( + driver_name: &str, + config: &Config, + file: Option<&config_file::ConfigFile>, +) -> Result { + let name = openshell_core::config::normalize_compute_driver_name(driver_name) + .map_err(Error::config)?; + let driver_kind = builtin_compute_driver(&name); + if let Some(socket_path) = config.compute_driver_endpoints.get(&name) { + if driver_kind.is_some() { + return Err(Error::config(format!( + "compute driver '{name}' is a reserved built-in driver and cannot be selected with a socket endpoint" + ))); + } + return Ok(ConfiguredComputeDriver::Remote( + RemoteComputeDriverSelection { + name, + socket_path: socket_path.clone(), + }, + )); + } + + if let Some(kind) = driver_kind { + return Ok(ConfiguredComputeDriver::Builtin(kind)); + } + + let socket_path = remote_driver_socket_from_file(&name, file)?; + Ok(ConfiguredComputeDriver::Remote( + RemoteComputeDriverSelection { name, socket_path }, + )) +} + +fn builtin_compute_driver(name: &str) -> Option { + name.parse().ok() +} + +fn remote_driver_socket_from_file( + name: &str, + file: Option<&config_file::ConfigFile>, +) -> Result { + let Some(file) = file else { + return Err(Error::config(format!( + "compute driver '{name}' is not a built-in driver; configure [openshell.drivers.{name}].socket_path or pass --drivers {name} with --compute-driver-socket" + ))); + }; + let Some(raw) = file.openshell.drivers.get(name) else { + return Err(Error::config(format!( + "compute driver '{name}' is not a built-in driver; configure [openshell.drivers.{name}].socket_path" + ))); + }; + let config = raw + .clone() + .try_into::() + .map_err(|err| { + Error::config(format!( + "invalid [openshell.drivers.{name}] table for remote compute driver: {err}" + )) + })?; + Ok(config.socket_path) +} + fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config, driver: ComputeDriverKind) -> bool { matches!(driver, ComputeDriverKind::Kubernetes) && config @@ -916,7 +1027,7 @@ fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config, driver: Compu #[cfg(test)] mod tests { use super::{ - ConnectionProtocol, MultiplexService, ServerState, TlsAcceptor, + ConfiguredComputeDriver, ConnectionProtocol, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, classify_initial_bytes, configured_compute_driver, gateway_listener_addresses, is_benign_tls_handshake_failure, kubernetes_config_for_k8s_sa_bootstrap, kubernetes_sandbox_jwt_expiry_disabled, @@ -928,6 +1039,7 @@ mod tests { }; use std::io::{Error, ErrorKind}; use std::net::SocketAddr; + use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use tempfile::{TempDir, tempdir}; @@ -1228,14 +1340,14 @@ mod tests { #[test] fn configured_compute_driver_triggers_auto_detection_when_empty() { - let config = Config::new(None).with_compute_drivers([]); + let config = Config::new(None).with_compute_drivers(std::iter::empty::()); // Empty drivers triggers auto-detection, which may return Some or None // depending on the environment. This test verifies the auto-detection path // is taken rather than immediately returning an error. - let result = configured_compute_driver(&config); + let result = configured_compute_driver(&config, None); // Either we get a detected driver or an error about none being detected. match result { - Ok(driver) => { + Ok(ConfiguredComputeDriver::Builtin(driver)) => { assert!( matches!( driver, @@ -1246,6 +1358,9 @@ mod tests { "auto-detected unexpected driver: {driver:?}" ); } + Ok(ConfiguredComputeDriver::Remote(remote)) => { + panic!("auto-detection returned remote driver: {remote:?}"); + } Err(e) => { assert!( e.to_string() @@ -1260,7 +1375,7 @@ mod tests { fn configured_compute_driver_rejects_multiple_entries() { let config = Config::new(None) .with_compute_drivers([ComputeDriverKind::Kubernetes, ComputeDriverKind::Podman]); - let err = configured_compute_driver(&config).unwrap_err(); + let err = configured_compute_driver(&config, None).unwrap_err(); assert!( err.to_string() .contains("multiple compute drivers are not supported yet") @@ -1271,27 +1386,90 @@ mod tests { #[test] fn configured_compute_driver_accepts_podman() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); - assert_eq!( - configured_compute_driver(&config).unwrap(), - ComputeDriverKind::Podman - ); + let driver = configured_compute_driver(&config, None).unwrap(); + assert!(matches!( + driver, + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) + )); } #[test] fn configured_compute_driver_accepts_vm() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); - assert_eq!( - configured_compute_driver(&config).unwrap(), - ComputeDriverKind::Vm - ); + let driver = configured_compute_driver(&config, None).unwrap(); + assert!(matches!( + driver, + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) + )); } #[test] fn configured_compute_driver_accepts_docker() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Docker]); - assert_eq!( - configured_compute_driver(&config).unwrap(), - ComputeDriverKind::Docker + let driver = configured_compute_driver(&config, None).unwrap(); + assert!(matches!( + driver, + ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) + )); + } + + #[test] + fn configured_compute_driver_resolves_named_remote_from_file() { + let file: super::config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +compute_drivers = ["kyma"] + +[openshell.drivers.kyma] +socket_path = "/run/openshell/kyma.sock" +"#, + ) + .unwrap(); + let config = Config::new(None).with_compute_drivers(["kyma"]); + + let driver = configured_compute_driver(&config, Some(&file)).unwrap(); + + match driver { + ConfiguredComputeDriver::Remote(remote) => { + assert_eq!(remote.name, "kyma"); + assert_eq!( + remote.socket_path, + PathBuf::from("/run/openshell/kyma.sock") + ); + } + ConfiguredComputeDriver::Builtin(other) => { + panic!("expected remote driver, got builtin driver {other:?}") + } + } + } + + #[test] + fn configured_compute_driver_rejects_vm_endpoint_from_config() { + let config = Config::new(None) + .with_compute_drivers([ComputeDriverKind::Vm]) + .with_compute_driver_endpoint("vm", "/run/openshell/vm.sock"); + + let err = configured_compute_driver(&config, None).unwrap_err(); + + assert!( + err.to_string() + .contains("reserved built-in driver and cannot be selected with a socket endpoint"), + "unexpected error: {err}" + ); + } + + #[test] + fn configured_compute_driver_rejects_builtin_endpoint() { + let config = Config::new(None) + .with_compute_drivers([ComputeDriverKind::Docker]) + .with_compute_driver_endpoint("docker", "/run/openshell/docker.sock"); + + let err = configured_compute_driver(&config, None).unwrap_err(); + + assert!( + err.to_string() + .contains("cannot be selected with a socket endpoint"), + "unexpected error: {err}" ); } diff --git a/crates/openshell-server/src/test_support.rs b/crates/openshell-server/src/test_support.rs new file mode 100644 index 0000000000..9cd80d6ed8 --- /dev/null +++ b/crates/openshell-server/src/test_support.rs @@ -0,0 +1,311 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Test fixtures for exercising gateway integration points. + +use futures::{Stream, stream}; +#[cfg(unix)] +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::proto::compute::v1::{ + CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, + DriverSandbox, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, + GetSandboxResponse, ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, + StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, + WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, +}; +use std::collections::HashMap; +#[cfg(unix)] +use std::io; +#[cfg(unix)] +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +#[cfg(unix)] +use std::task::{Context, Poll}; +#[cfg(unix)] +use tokio::net::{UnixListener, UnixStream}; +#[cfg(unix)] +use tokio::task::JoinHandle; +use tonic::{Request, Response, Status}; + +type WatchStream = Pin> + Send>>; + +#[derive(Debug, Clone, PartialEq)] +pub enum FakeComputeDriverCall { + GetCapabilities, + ValidateSandboxCreate { + sandbox: Option, + }, + GetSandbox { + sandbox_id: String, + sandbox_name: String, + }, + ListSandboxes, + CreateSandbox { + sandbox: Option, + }, + StopSandbox { + sandbox_id: String, + sandbox_name: String, + }, + DeleteSandbox { + sandbox_id: String, + sandbox_name: String, + }, + WatchSandboxes, +} + +#[derive(Debug, Clone)] +pub struct FakeComputeDriver { + state: Arc>, +} + +#[derive(Debug)] +struct FakeComputeDriverState { + driver_name: String, + driver_version: String, + default_image: String, + sandboxes: HashMap, + calls: Vec, +} + +impl Default for FakeComputeDriver { + fn default() -> Self { + Self::new() + } +} + +impl FakeComputeDriver { + #[must_use] + pub fn new() -> Self { + Self { + state: Arc::new(Mutex::new(FakeComputeDriverState { + driver_name: "fake-compute-driver".to_string(), + driver_version: "test".to_string(), + default_image: "openshell/sandbox:test".to_string(), + sandboxes: HashMap::new(), + calls: Vec::new(), + })), + } + } + + #[must_use] + pub fn with_driver_name(self, driver_name: impl Into) -> Self { + self.with_state(|state| state.driver_name = driver_name.into()); + self + } + + #[must_use] + pub fn with_driver_version(self, driver_version: impl Into) -> Self { + self.with_state(|state| state.driver_version = driver_version.into()); + self + } + + #[must_use] + pub fn with_default_image(self, default_image: impl Into) -> Self { + self.with_state(|state| state.default_image = default_image.into()); + self + } + + #[must_use] + pub fn calls(&self) -> Vec { + self.with_state(|state| state.calls.clone()) + } + + pub fn clear_calls(&self) { + self.with_state(|state| state.calls.clear()); + } + + #[cfg(unix)] + pub fn serve_uds( + &self, + socket_path: impl AsRef, + ) -> io::Result { + let socket_path = socket_path.as_ref().to_path_buf(); + let listener = UnixListener::bind(&socket_path)?; + let driver = self.clone(); + let task = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming(UnixIncoming { listener }) + .await + }); + Ok(FakeComputeDriverServerHandle { socket_path, task }) + } + + fn with_state(&self, f: impl FnOnce(&mut FakeComputeDriverState) -> R) -> R { + let mut state = self + .state + .lock() + .expect("fake compute driver state poisoned"); + f(&mut state) + } +} + +#[cfg(unix)] +#[derive(Debug)] +pub struct FakeComputeDriverServerHandle { + socket_path: PathBuf, + task: JoinHandle>, +} + +#[cfg(unix)] +impl Drop for FakeComputeDriverServerHandle { + fn drop(&mut self) { + self.task.abort(); + let _ = std::fs::remove_file(&self.socket_path); + } +} + +#[cfg(unix)] +struct UnixIncoming { + listener: UnixListener, +} + +#[cfg(unix)] +impl Stream for UnixIncoming { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut().listener.poll_accept(cx) { + Poll::Ready(Ok((stream, _addr))) => Poll::Ready(Some(Ok(stream))), + Poll::Ready(Err(err)) => Poll::Ready(Some(Err(err))), + Poll::Pending => Poll::Pending, + } + } +} + +#[tonic::async_trait] +impl ComputeDriver for FakeComputeDriver { + type WatchSandboxesStream = WatchStream; + + async fn get_capabilities( + &self, + _request: Request, + ) -> Result, Status> { + let response = self.with_state(|state| { + state.calls.push(FakeComputeDriverCall::GetCapabilities); + GetCapabilitiesResponse { + driver_name: state.driver_name.clone(), + driver_version: state.driver_version.clone(), + default_image: state.default_image.clone(), + } + }); + Ok(Response::new(response)) + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request.into_inner().sandbox; + self.with_state(|state| { + state + .calls + .push(FakeComputeDriverCall::ValidateSandboxCreate { sandbox }); + }); + Ok(Response::new(ValidateSandboxCreateResponse {})) + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let sandbox = self.with_state(|state| { + state.calls.push(FakeComputeDriverCall::GetSandbox { + sandbox_id: request.sandbox_id.clone(), + sandbox_name: request.sandbox_name.clone(), + }); + state + .sandboxes + .values() + .find(|sandbox| { + (!request.sandbox_id.is_empty() && sandbox.id == request.sandbox_id) + || (!request.sandbox_name.is_empty() + && sandbox.name == request.sandbox_name) + }) + .cloned() + }); + let sandbox = sandbox.ok_or_else(|| Status::not_found("sandbox not found"))?; + Ok(Response::new(GetSandboxResponse { + sandbox: Some(sandbox), + })) + } + + async fn list_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + let sandboxes = self.with_state(|state| { + state.calls.push(FakeComputeDriverCall::ListSandboxes); + state.sandboxes.values().cloned().collect() + }); + Ok(Response::new(ListSandboxesResponse { sandboxes })) + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request.into_inner().sandbox; + self.with_state(|state| { + if let Some(sandbox) = sandbox.as_ref() { + state.sandboxes.insert(sandbox.id.clone(), sandbox.clone()); + } + state + .calls + .push(FakeComputeDriverCall::CreateSandbox { sandbox }); + }); + Ok(Response::new(CreateSandboxResponse {})) + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + self.with_state(|state| { + state.calls.push(FakeComputeDriverCall::StopSandbox { + sandbox_id: request.sandbox_id, + sandbox_name: request.sandbox_name, + }); + }); + Ok(Response::new(StopSandboxResponse {})) + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let deleted = self.with_state(|state| { + state.calls.push(FakeComputeDriverCall::DeleteSandbox { + sandbox_id: request.sandbox_id.clone(), + sandbox_name: request.sandbox_name.clone(), + }); + if request.sandbox_id.is_empty() { + let Some(id) = state + .sandboxes + .iter() + .find(|(_, sandbox)| sandbox.name == request.sandbox_name) + .map(|(id, _)| id.clone()) + else { + return false; + }; + state.sandboxes.remove(&id).is_some() + } else { + state.sandboxes.remove(&request.sandbox_id).is_some() + } + }); + Ok(Response::new(DeleteSandboxResponse { deleted })) + } + + async fn watch_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + self.with_state(|state| state.calls.push(FakeComputeDriverCall::WatchSandboxes)); + Ok(Response::new(Box::pin(stream::empty()))) + } +} diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d820a131dc..2aaa6e7b06 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -307,3 +307,24 @@ guest_tls_ca = "/var/lib/openshell/guest-tls/ca.pem" guest_tls_cert = "/var/lib/openshell/guest-tls/client.pem" guest_tls_key = "/var/lib/openshell/guest-tls/client-key.pem" ``` + +### Extension Driver + +Extension drivers run outside the gateway and expose the +`compute_driver.proto` gRPC service on a Unix socket. Use a non-reserved driver +name; built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot +be selected through unmanaged socket endpoints. The selected driver name is the +key used for driver-owned sandbox config such as `template.driver_config.`. + +```toml +[openshell] +version = 1 + +[openshell.gateway] +bind_address = "127.0.0.1:17670" +log_level = "info" +compute_drivers = ["kyma"] + +[openshell.drivers.kyma] +socket_path = "/run/openshell/kyma-compute-driver.sock" +``` diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index c81afd66d0..9e944715cf 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -21,7 +21,9 @@ Configure the compute driver on the gateway. Current releases accept one driver compute_drivers = ["docker"] ``` -Supported values are `docker`, `podman`, `kubernetes`, and `vm`. +Reserved built-in values are `docker`, `podman`, `kubernetes`, and `vm`. +Non-reserved names select an extension driver and require a +`socket_path` in `[openshell.drivers.]`. When `compute_drivers` is unset, the gateway auto-detects Kubernetes, then Podman, then Docker by CLI availability or a local Unix socket. The VM driver is never auto-detected; configure it explicitly with `compute_drivers = ["vm"]` or set `OPENSHELL_DRIVERS=vm` in the launch environment. @@ -29,10 +31,33 @@ Common gateway options: | Gateway TOML option | Description | |---|---| -| `compute_drivers = [""]` | Select the compute driver. Supported values are `docker`, `podman`, `kubernetes`, and `vm`. | +| `compute_drivers = [""]` | Select the compute driver. Built-in values are `docker`, `podman`, `kubernetes`, and `vm`; custom names require `[openshell.drivers.].socket_path`. | Set driver-specific values such as sandbox images, callback endpoints, network names, TLS material, and VM sizing in the gateway TOML file. See the [Gateway Configuration File](./gateway-config) reference for the full `[openshell.drivers.]` schema. +Extension drivers use the same `compute_driver.proto` gRPC surface as the +managed VM driver. For an out-of-tree driver, choose a driver name and point +the gateway at the Unix socket the operator has already provisioned: + +```toml +[openshell.gateway] +compute_drivers = ["kyma"] + +[openshell.drivers.kyma] +socket_path = "/run/openshell/kyma.sock" +``` + +For a launch-time socket override, pass the same non-reserved driver name with +the socket path: + +```shell +openshell-gateway --drivers kyma --compute-driver-socket /run/openshell/kyma.sock +``` + +The gateway does not spawn, supervise, or delete extension drivers. The +operator must protect the socket so only the gateway uid can access it. +Reserved built-in names cannot be selected through unmanaged socket endpoints. + Sandbox create supports `--cpu` and `--memory` for per-sandbox compute sizing. Docker and Podman apply them as runtime limits. Kubernetes applies them as both container requests and limits. The VM driver accepts the fields but currently @@ -240,7 +265,7 @@ compute_drivers = ["vm"] For a launch-time override, set `OPENSHELL_DRIVERS=vm` in the gateway environment and restart the service. -Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. +Configure VM driver values such as `grpc_endpoint`, `driver_dir`, `state_dir`, `default_image`, `bootstrap_image`, `vcpus`, `mem_mib`, `overlay_disk_mib`, `krun_log_level`, and `guest_tls_*` in `[openshell.drivers.vm]`. The VM `state_dir` stores overlay disks, console logs, runtime state, image-rootfs cache, and the private `run/compute-driver.sock` socket. The VM socket path is managed by the gateway and is not configurable through remote endpoint settings. The gateway starts `openshell-driver-vm` over a private Unix socket and passes its process ID so the driver can reject unexpected local clients. The driver's standalone TCP listener is disabled unless `--allow-unauthenticated-tcp` is set for local development. From e3382cb44747b5206ace2e7251a28f9f0bee4bf6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 26 Jun 2026 15:59:15 +0200 Subject: [PATCH 09/20] fix(server): update driver spec test argument (#2022) * fix(server): update driver spec test argument Signed-off-by: Evan Lezar * test(server): stabilize concurrent lease steal test Signed-off-by: Evan Lezar --------- Signed-off-by: Evan Lezar --- crates/openshell-server/src/compute/lease.rs | 8 ++++++-- crates/openshell-server/src/compute/mod.rs | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/openshell-server/src/compute/lease.rs b/crates/openshell-server/src/compute/lease.rs index b197395e4e..a2eac70625 100644 --- a/crates/openshell-server/src/compute/lease.rs +++ b/crates/openshell-server/src/compute/lease.rs @@ -480,14 +480,18 @@ mod tests { #[tokio::test] async fn concurrent_steal_exactly_one_wins() { let store = test_store().await; - let l_holder = lease(store.clone(), "holder", Duration::ZERO); + let test_ttl = Duration::from_millis(100); + let expiry_slack = Duration::from_millis(25); + + let l_holder = lease(store.clone(), "holder", test_ttl); let _guard = l_holder.try_acquire().await.unwrap(); + tokio::time::sleep(test_ttl + expiry_slack).await; let mut tasks = Vec::new(); for i in 0..5 { let s = store.clone(); tasks.push(tokio::spawn(async move { - let l = lease(s, &format!("standby-{i}"), Duration::ZERO); + let l = lease(s, &format!("standby-{i}"), test_ttl); l.try_steal_expired().await })); } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 10953f1bf6..368c9c7e58 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -2172,8 +2172,8 @@ mod tests { ..Default::default() }; - let driver = - driver_sandbox_spec_from_public(&public, None).expect("driver spec should map"); + let driver = driver_sandbox_spec_from_public(&public, "test-driver") + .expect("driver spec should map"); let gpu = driver .resource_requirements From 7ea471cd5a81ae3205d2fbed5bdc22e7f8447b94 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 26 Jun 2026 16:14:08 +0200 Subject: [PATCH 10/20] chore(deps): remove unused regorus yaml feature (#2021) Signed-off-by: Evan Lezar --- Cargo.lock | 1 - crates/openshell-prover/src/policy.rs | 5 +++-- crates/openshell-supervisor-network/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f693acd66b..081ffe6be7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4674,7 +4674,6 @@ dependencies = [ "rand 0.9.4", "serde", "serde_json", - "serde_yaml", "spin", "thiserror 2.0.18", ] diff --git a/crates/openshell-prover/src/policy.rs b/crates/openshell-prover/src/policy.rs index 8aea4b7d00..aa40d07560 100644 --- a/crates/openshell-prover/src/policy.rs +++ b/crates/openshell-prover/src/policy.rs @@ -12,6 +12,7 @@ use std::path::Path; use miette::{IntoDiagnostic, Result, WrapErr}; use serde::Deserialize; +use serde::de::IgnoredAny; // --------------------------------------------------------------------------- // Policy intent @@ -60,10 +61,10 @@ struct PolicyFile { // Ignored fields the prover does not need. #[serde(default)] #[allow(dead_code)] - landlock: Option, + landlock: Option, #[serde(default)] #[allow(dead_code)] - process: Option, + process: Option, } #[derive(Debug, Deserialize)] diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 71febf0af9..0f19d1ff30 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -25,7 +25,7 @@ hex = "0.4" ipnet = "2" miette = { workspace = true } rcgen = { workspace = true } -regorus = { version = "0.9", default-features = false, features = ["std", "arc", "glob", "yaml"] } +regorus = { version = "0.9", default-features = false, features = ["std", "arc", "glob"] } reqwest = { workspace = true } rustls = { workspace = true } rustls-pemfile = { workspace = true } From a242f84bb367d6df7d4d133e95a93857406c67f7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 26 Jun 2026 16:23:44 +0200 Subject: [PATCH 11/20] chore(gitignore): ignore nix result links (#2020) Signed-off-by: Evan Lezar --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a3d6137757..34b24bf768 100644 --- a/.gitignore +++ b/.gitignore @@ -225,3 +225,7 @@ rfc.md # Markdown/mermaid lint tooling deps scripts/lint-mermaid/node_modules/ + +# Nix +/result +/result-* From b855d8d8721871da0cfe1fed329f96d5f258c27a Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Fri, 26 Jun 2026 08:28:35 -0700 Subject: [PATCH 12/20] fix(policy): reserve provider rule namespace (#1991) Closes #1982 Reject user-authored _provider_* network policy keys at gateway boundaries, strip provider-derived rules from sandbox sync, and document the round-trip workflow. Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-cli/src/main.rs | 41 ++- crates/openshell-cli/src/run.rs | 79 +++-- .../sandbox_name_fallback_integration.rs | 95 +++++- crates/openshell-policy/src/compose.rs | 89 ++++- crates/openshell-policy/src/lib.rs | 5 +- crates/openshell-policy/src/merge.rs | 6 +- crates/openshell-sandbox/src/lib.rs | 107 +++++- crates/openshell-server/src/grpc/policy.rs | 309 +++++++++++++++++- crates/openshell-server/src/grpc/sandbox.rs | 36 +- .../openshell-server/src/grpc/validation.rs | 53 +++ docs/sandboxes/policies.mdx | 6 +- docs/sandboxes/providers-v2.mdx | 16 +- 12 files changed, 754 insertions(+), 88 deletions(-) diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 1e72f0ba5c..e02c7c9cd5 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1693,10 +1693,14 @@ enum PolicyCommands { #[arg(long = "rev", default_value_t = 0)] rev: u32, - /// Include the full policy payload. - #[arg(long)] + /// Include the effective policy payload, including provider-composed entries. + #[arg(long, conflicts_with = "base")] full: bool, + /// Include the base policy payload without provider-composed entries. + #[arg(long)] + base: bool, + /// Output format. #[arg(short = 'o', long = "output", value_enum, default_value_t = PolicyGetOutput::Table)] output: PolicyGetOutput, @@ -2378,14 +2382,16 @@ async fn main() -> Result<()> { name, rev, full, + base, output, global, } => { + let view = run::PolicyGetView::from_flags(base, full); if global { run::sandbox_policy_get_global( &ctx.endpoint, rev, - full, + view, output.as_str(), &tls, ) @@ -2396,7 +2402,7 @@ async fn main() -> Result<()> { &ctx.endpoint, &name, rev, - full, + view, output.as_str(), &tls, ) @@ -4366,17 +4372,42 @@ mod tests { Some(Commands::Policy { command: Some(PolicyCommands::Get { - name, full, output, .. + name, + full, + base, + output, + .. }), }) => { assert_eq!(name.as_deref(), Some("my-sandbox")); assert!(full); + assert!(!base); assert!(matches!(output, PolicyGetOutput::Json)); } other => panic!("expected policy get command, got: {other:?}"), } } + #[test] + fn policy_get_base_output_parses() { + let cli = Cli::try_parse_from(["openshell", "policy", "get", "my-sandbox", "--base"]) + .expect("policy get --base should parse"); + + match cli.command { + Some(Commands::Policy { + command: + Some(PolicyCommands::Get { + name, full, base, .. + }), + }) => { + assert_eq!(name.as_deref(), Some("my-sandbox")); + assert!(!full); + assert!(base); + } + other => panic!("expected policy get command, got: {other:?}"), + } + } + #[test] fn policy_delete_global_parses() { let cli = Cli::try_parse_from(["openshell", "policy", "delete", "--global", "--yes"]) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index fee2641d8e..f56bb71512 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -63,6 +63,7 @@ use openshell_providers::{ profile_to_json, profile_to_yaml, profiles_to_json, profiles_to_yaml, }; use owo_colors::OwoColorize; +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::io::{ErrorKind, IsTerminal, Read, Write}; use std::path::{Path, PathBuf}; @@ -80,6 +81,27 @@ pub use openshell_core::forward::{ ForwardSpec, find_forward_by_port, list_forwards, stop_forward, stop_forwards_for_sandbox, }; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum PolicyGetView { + Metadata, + Base, + Full, +} + +impl PolicyGetView { + pub fn from_flags(base: bool, full: bool) -> Self { + match (base, full) { + (true, _) => Self::Base, + (false, true) => Self::Full, + (false, false) => Self::Metadata, + } + } + + fn includes_policy(self) -> bool { + matches!(self, Self::Base | Self::Full) + } +} + #[derive(Debug, PartialEq, Eq)] enum SandboxUploadPlan { GitAware { @@ -6919,7 +6941,7 @@ pub async fn sandbox_policy_get( server: &str, name: &str, version: u32, - full: bool, + view: PolicyGetView, output: &str, tls: &TlsOptions, ) -> Result<()> { @@ -6929,7 +6951,7 @@ pub async fn sandbox_policy_get( server, name, version, - full, + view, output, tls, (&mut stdout, &mut stderr), @@ -6953,7 +6975,7 @@ pub async fn sandbox_policy_get_to_writer( server: &str, name: &str, version: u32, - full: bool, + view: PolicyGetView, output: &str, tls: &TlsOptions, writers: (&mut W, &mut E), @@ -6963,7 +6985,7 @@ where E: Write + Send, { if version == 0 { - return sandbox_policy_get_effective_to_writer(server, name, full, output, tls, writers) + return sandbox_policy_get_effective_to_writer(server, name, view, output, tls, writers) .await; } @@ -6990,7 +7012,7 @@ where Some(inner.active_version), &rev, status, - full, + view, )?; writeln!( stdout, @@ -7018,10 +7040,11 @@ where writeln!(stdout, "Error: {}", rev.load_error).into_diagnostic()?; } - if full { + if view.includes_policy() { if let Some(ref policy) = rev.policy { writeln!(stdout, "---").into_diagnostic()?; - let yaml_str = openshell_policy::serialize_sandbox_policy(policy) + let policy = policy_for_view(policy, view); + let yaml_str = openshell_policy::serialize_sandbox_policy(policy.as_ref()) .wrap_err("failed to serialize policy to YAML")?; write!(stdout, "{yaml_str}").into_diagnostic()?; } else { @@ -7039,7 +7062,7 @@ where async fn sandbox_policy_get_effective_to_writer( server: &str, name: &str, - full: bool, + view: PolicyGetView, output: &str, tls: &TlsOptions, writers: (&mut W, &mut E), @@ -7112,10 +7135,11 @@ where serde_json::json!(config.global_policy_version), ); } - if full { + if view.includes_policy() { + let policy = policy_for_view(policy, view); obj.insert( "policy".to_string(), - openshell_policy::sandbox_policy_to_json_value(policy)?, + openshell_policy::sandbox_policy_to_json_value(policy.as_ref())?, ); } writeln!( @@ -7135,9 +7159,10 @@ where writeln!(stdout, "Global: {}", config.global_policy_version) .into_diagnostic()?; } - if full { + if view.includes_policy() { writeln!(stdout, "---").into_diagnostic()?; - let yaml_str = openshell_policy::serialize_sandbox_policy(policy) + let policy = policy_for_view(policy, view); + let yaml_str = openshell_policy::serialize_sandbox_policy(policy.as_ref()) .wrap_err("failed to serialize policy to YAML")?; write!(stdout, "{yaml_str}").into_diagnostic()?; } @@ -7151,7 +7176,7 @@ where pub async fn sandbox_policy_get_global( server: &str, version: u32, - full: bool, + view: PolicyGetView, output: &str, tls: &TlsOptions, ) -> Result<()> { @@ -7171,7 +7196,7 @@ pub async fn sandbox_policy_get_global( let status = PolicyStatus::try_from(rev.status).unwrap_or(PolicyStatus::Unspecified); match output { "json" => { - let obj = policy_revision_to_json("global", None, None, &rev, status, full)?; + let obj = policy_revision_to_json("global", None, None, &rev, status, view)?; println!("{}", serde_json::to_string_pretty(&obj).into_diagnostic()?); return Ok(()); } @@ -7190,10 +7215,11 @@ pub async fn sandbox_policy_get_global( println!("Loaded: {} ms", rev.loaded_at_ms); } - if full { + if view.includes_policy() { if let Some(ref policy) = rev.policy { println!("---"); - let yaml_str = openshell_policy::serialize_sandbox_policy(policy) + let policy = policy_for_view(policy, view); + let yaml_str = openshell_policy::serialize_sandbox_policy(policy.as_ref()) .wrap_err("failed to serialize policy to YAML")?; print!("{yaml_str}"); } else { @@ -7223,7 +7249,7 @@ fn policy_revision_to_json( active_version: Option, rev: &openshell_core::proto::SandboxPolicyRevision, status: PolicyStatus, - full: bool, + view: PolicyGetView, ) -> Result { let mut obj = serde_json::Map::new(); obj.insert("scope".to_string(), serde_json::json!(scope)); @@ -7257,9 +7283,12 @@ fn policy_revision_to_json( if !rev.load_error.is_empty() { obj.insert("load_error".to_string(), serde_json::json!(rev.load_error)); } - if full { + if view.includes_policy() { let policy = match rev.policy.as_ref() { - Some(policy) => openshell_policy::sandbox_policy_to_json_value(policy)?, + Some(policy) => { + let policy = policy_for_view(policy, view); + openshell_policy::sandbox_policy_to_json_value(policy.as_ref())? + } None => serde_json::Value::Null, }; obj.insert("policy".to_string(), policy); @@ -7267,6 +7296,18 @@ fn policy_revision_to_json( Ok(serde_json::Value::Object(obj)) } +fn policy_for_view(policy: &SandboxPolicy, view: PolicyGetView) -> Cow<'_, SandboxPolicy> { + if view != PolicyGetView::Base { + return Cow::Borrowed(policy); + } + + let mut base_policy = policy.clone(); + base_policy + .network_policies + .retain(|name, _| !openshell_policy::is_provider_rule_name(name)); + Cow::Owned(base_policy) +} + pub async fn sandbox_policy_list( server: &str, name: &str, diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index d4052ff683..8e799f821e 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -132,21 +132,39 @@ impl OpenShell for TestOpenShell { Ok(Response::new(GetSandboxConfigResponse { policy: Some(SandboxPolicy { version: 9, - network_policies: std::iter::once(( - "_provider_api".to_string(), - NetworkPolicyRule { - name: "_provider_api".to_string(), - endpoints: vec![NetworkEndpoint { - host: "api.provider.example.com".to_string(), - port: 443, - protocol: "rest".to_string(), - enforcement: "enforce".to_string(), - access: "read-only".to_string(), + network_policies: [ + ( + "user_api".to_string(), + NetworkPolicyRule { + name: "user_api".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.user.example.com".to_string(), + port: 443, + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + access: "read-only".to_string(), + ..Default::default() + }], ..Default::default() - }], - ..Default::default() - }, - )) + }, + ), + ( + "_provider_api".to_string(), + NetworkPolicyRule { + name: "_provider_api".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.provider.example.com".to_string(), + port: 443, + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + access: "read-only".to_string(), + ..Default::default() + }], + ..Default::default() + }, + ), + ] + .into_iter() .collect(), ..Default::default() }), @@ -675,7 +693,7 @@ async fn policy_get_full_json_cli_prints_policy_payload() { &ts.endpoint, "my-sandbox", 0, - true, + run::PolicyGetView::Full, "json", &ts.tls, (&mut stdout, &mut stderr), @@ -707,6 +725,51 @@ async fn policy_get_full_json_cli_prints_policy_payload() { json["policy"]["network_policies"]["_provider_api"]["endpoints"][0]["host"], "api.provider.example.com" ); + assert_eq!( + json["policy"]["network_policies"]["user_api"]["endpoints"][0]["host"], + "api.user.example.com" + ); +} + +#[tokio::test] +async fn policy_get_base_json_cli_prints_round_trippable_policy_payload() { + let ts = run_server().await; + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + + run::sandbox_policy_get_to_writer( + &ts.endpoint, + "my-sandbox", + 0, + run::PolicyGetView::Base, + "json", + &ts.tls, + (&mut stdout, &mut stderr), + ) + .await + .expect("policy get --base should succeed"); + + assert!( + stderr.is_empty(), + "policy get --base should not print stderr: {}", + String::from_utf8_lossy(&stderr) + ); + + let json: serde_json::Value = + serde_json::from_slice(&stdout).expect("stdout should be valid JSON"); + assert_eq!(json["scope"], "sandbox"); + assert_eq!(json["sandbox"], "my-sandbox"); + assert_eq!(json["status"], "effective"); + assert!( + json["policy"]["network_policies"] + .get("_provider_api") + .is_none(), + "base policy output must omit provider-composed policy entries" + ); + assert_eq!( + json["policy"]["network_policies"]["user_api"]["endpoints"][0]["host"], + "api.user.example.com" + ); } #[tokio::test] @@ -719,7 +782,7 @@ async fn policy_get_explicit_revision_uses_stored_policy_status() { &ts.endpoint, "my-sandbox", 3, - true, + run::PolicyGetView::Full, "json", &ts.tls, (&mut stdout, &mut stderr), diff --git a/crates/openshell-policy/src/compose.rs b/crates/openshell-policy/src/compose.rs index 427fa4cda5..7ca8584d9d 100644 --- a/crates/openshell-policy/src/compose.rs +++ b/crates/openshell-policy/src/compose.rs @@ -5,12 +5,19 @@ use openshell_core::proto::{NetworkPolicyRule, SandboxPolicy}; +pub const PROVIDER_RULE_NAME_PREFIX: &str = "_provider_"; + #[derive(Debug, Clone, PartialEq)] pub struct ProviderPolicyLayer { pub rule_name: String, pub rule: NetworkPolicyRule, } +#[must_use] +pub fn is_provider_rule_name(rule_name: &str) -> bool { + rule_name.starts_with(PROVIDER_RULE_NAME_PREFIX) +} + #[must_use] pub fn provider_rule_name(provider_name: &str) -> String { let sanitized = provider_name @@ -27,19 +34,27 @@ pub fn provider_rule_name(provider_name: &str) -> String { .to_string(); if sanitized.is_empty() { - "_provider_unnamed".to_string() + format!("{PROVIDER_RULE_NAME_PREFIX}unnamed") } else { - format!("_provider_{sanitized}") + format!("{PROVIDER_RULE_NAME_PREFIX}{sanitized}") } } +pub fn strip_provider_rule_names(policy: &mut SandboxPolicy) -> bool { + let original_len = policy.network_policies.len(); + policy + .network_policies + .retain(|key, _| !is_provider_rule_name(key)); + policy.network_policies.len() != original_len +} + /// Compose a normal sandbox policy from user-authored policy plus provider /// policy layers. /// /// The returned policy is derived data. It preserves the source policy's /// static fields and user-authored network policies, then concatenates each -/// provider rule under a reserved `_provider_*` key. Existing user keys are not -/// overwritten; a numeric suffix is added if needed. +/// provider rule under a reserved `_provider_*` key. Existing keys are not +/// overwritten; a numeric suffix is added if provider rule names collide. #[must_use] pub fn compose_effective_policy( source_policy: &SandboxPolicy, @@ -76,7 +91,10 @@ fn unique_provider_rule_key(policy: &SandboxPolicy, preferred: &str) -> String { #[cfg(test)] mod tests { - use super::{ProviderPolicyLayer, compose_effective_policy, provider_rule_name}; + use super::{ + PROVIDER_RULE_NAME_PREFIX, ProviderPolicyLayer, compose_effective_policy, + is_provider_rule_name, provider_rule_name, strip_provider_rule_names, + }; use openshell_core::proto::{NetworkEndpoint, NetworkPolicyRule, SandboxPolicy}; fn rule(name: &str, host: &str) -> NetworkPolicyRule { @@ -107,6 +125,38 @@ mod tests { assert_eq!(provider_rule_name("..."), "_provider_unnamed"); } + #[test] + fn provider_rule_name_prefix_identifies_reserved_keys() { + assert_eq!(PROVIDER_RULE_NAME_PREFIX, "_provider_"); + assert!(is_provider_rule_name("_provider_work_github")); + assert!(is_provider_rule_name("_provider_work_github_2")); + assert!(is_provider_rule_name("_provider_")); + assert!(!is_provider_rule_name("provider_work_github")); + assert!(!is_provider_rule_name("custom_provider_work_github")); + } + + #[test] + fn strip_provider_rule_names_removes_only_reserved_keys() { + let mut policy = SandboxPolicy::default(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + rule("_provider_work_github", "api.github.com"), + ); + policy.network_policies.insert( + "sandbox_only".to_string(), + rule("sandbox_only", "sandbox.example.com"), + ); + + assert!(strip_provider_rule_names(&mut policy)); + assert!( + !policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(policy.network_policies.contains_key("sandbox_only")); + assert!(!strip_provider_rule_names(&mut policy)); + } + #[test] fn compose_concatenates_provider_rules_without_overwriting_user_rules() { let mut source = SandboxPolicy::default(); @@ -114,17 +164,19 @@ mod tests { "custom_github".to_string(), rule("custom_github", "api.github.com"), ); - source.network_policies.insert( - "_provider_work_github".to_string(), - rule("_provider_work_github", "example.com"), - ); let effective = compose_effective_policy( &source, - &[ProviderPolicyLayer { - rule_name: "_provider_work_github".to_string(), - rule: rule("_provider_work_github", "github.com"), - }], + &[ + ProviderPolicyLayer { + rule_name: "_provider_work_github".to_string(), + rule: rule("_provider_work_github", "github.com"), + }, + ProviderPolicyLayer { + rule_name: "_provider_work_github".to_string(), + rule: rule("_provider_work_github", "github.example.com"), + }, + ], ); assert!(effective.network_policies.contains_key("custom_github")); @@ -138,7 +190,16 @@ mod tests { .network_policies .contains_key("_provider_work_github_2") ); - assert_eq!(source.network_policies.len(), 2); + assert_eq!( + effective + .network_policies + .get("custom_github") + .unwrap() + .endpoints[0] + .host, + "api.github.com" + ); + assert_eq!(source.network_policies.len(), 1); assert_eq!(effective.network_policies.len(), 3); } } diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index aaabbf9260..22af7093e4 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -24,7 +24,10 @@ use openshell_core::proto::{ }; use serde::{Deserialize, Serialize}; -pub use compose::{ProviderPolicyLayer, compose_effective_policy, provider_rule_name}; +pub use compose::{ + PROVIDER_RULE_NAME_PREFIX, ProviderPolicyLayer, compose_effective_policy, + is_provider_rule_name, provider_rule_name, strip_provider_rule_names, +}; pub use merge::{ PolicyMergeError, PolicyMergeOp, PolicyMergeResult, PolicyMergeWarning, generated_rule_name, merge_policy, policy_covers_rule, diff --git a/crates/openshell-policy/src/merge.rs b/crates/openshell-policy/src/merge.rs index f191cd2721..89c8bf8b36 100644 --- a/crates/openshell-policy/src/merge.rs +++ b/crates/openshell-policy/src/merge.rs @@ -7,6 +7,8 @@ use openshell_core::proto::{ L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, SandboxPolicy, }; +use crate::is_provider_rule_name; + #[derive(Debug, Clone, PartialEq)] pub enum PolicyMergeOp { AddRule { @@ -413,7 +415,7 @@ fn add_rule( let mut keys: Vec<_> = policy.network_policies.keys().cloned().collect(); keys.sort(); keys.into_iter() - .filter(|k| !k.starts_with("_provider_")) + .filter(|k| !is_provider_rule_name(k)) .find(|key| { policy .network_policies @@ -653,7 +655,7 @@ fn find_endpoint_mut<'a>( keys.sort(); let target_key = keys .into_iter() - .filter(|k| !k.starts_with("_provider_")) + .filter(|k| !is_provider_rule_name(k)) .find(|key| { policy.network_policies.get(key).is_some_and(|rule| { rule.endpoints diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 63ad1117cb..47550a9715 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -886,6 +886,23 @@ fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) modified } +fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { + openshell_policy::strip_provider_rule_names(proto) +} + +fn proto_sync_payload_for_enriched_policy( + proto: &openshell_core::proto::SandboxPolicy, + enriched: bool, +) -> Option { + if !enriched { + return None; + } + + let mut sync_policy = proto.clone(); + strip_proto_provider_policy_entries(&mut sync_policy); + Some(sync_policy) +} + /// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem /// paths required by proxy-mode sandboxes and GPU runtimes. Used for the /// local-file code path where no proto is available. @@ -1050,6 +1067,89 @@ mod baseline_tests { ); } + #[test] + fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + assert!(strip_proto_provider_policy_entries(&mut policy)); + assert!( + !policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(policy.network_policies.contains_key("sandbox_only")); + assert!(!strip_proto_provider_policy_entries(&mut policy)); + } + + #[test] + fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + + assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "provider-derived rules alone must not trigger sync or mutate runtime policy" + ); + } + + #[test] + fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { + let mut runtime_policy = openshell_policy::restrictive_default_policy(); + runtime_policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + runtime_policy.network_policies.insert( + "sandbox_only".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "sandbox_only".to_string(), + ..Default::default() + }, + ); + + let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) + .expect("enrichment should create a sync payload"); + + assert!( + runtime_policy + .network_policies + .contains_key("_provider_work_github"), + "runtime policy must retain provider-derived rules for OPA input" + ); + assert!( + !sync_policy + .network_policies + .contains_key("_provider_work_github") + ); + assert!(sync_policy.network_policies.contains_key("sandbox_only")); + } + #[test] fn proto_gpu_enrichment_promotes_proc_without_network_policy() { let mut policy = openshell_policy::restrictive_default_policy(); @@ -1298,6 +1398,7 @@ async fn load_policy( // Enrich before syncing so the gateway baseline includes // baseline paths from the start. enrich_proto_baseline_paths(&mut discovered); + strip_proto_provider_policy_entries(&mut discovered); let sandbox = sandbox.as_deref().ok_or_else(|| { miette::miette!( "Cannot sync discovered policy: sandbox not available.\n\ @@ -1322,11 +1423,11 @@ async fn load_policy( // sandboxes. If the policy was enriched, sync the updated version // back to the gateway so users can see the effective policy. let enriched = enrich_proto_baseline_paths(&mut proto_policy); - if enriched + let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); + if let Some(sync_policy) = sync_policy && let Some(sandbox_name) = sandbox.as_deref() && let Err(e) = - openshell_core::grpc_client::sync_policy(endpoint, sandbox_name, &proto_policy) - .await + openshell_core::grpc_client::sync_policy(endpoint, sandbox_name, &sync_policy).await { warn!( error = %e, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index b4fd093ea2..8b3b20bec2 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -71,7 +71,8 @@ use tonic::{Request, Response, Status}; use tracing::{debug, info, warn}; use super::validation::{ - level_matches, source_matches, validate_policy_safety, validate_static_fields_unchanged, + level_matches, source_matches, validate_no_reserved_provider_policy_keys, + validate_policy_safety, validate_static_fields_unchanged, }; use super::{MAX_PAGE_SIZE, StoredSettingValue, StoredSettings, clamp_limit}; use crate::persistence::current_time_ms; @@ -1507,6 +1508,7 @@ async fn handle_update_config_inner( Status::invalid_argument("policy is required for global policy update") })?; openshell_policy::ensure_sandbox_process_identity(&mut new_policy); + validate_no_reserved_provider_policy_keys(&new_policy)?; validate_policy_safety(&new_policy)?; let payload = new_policy.encode_to_vec(); @@ -1814,6 +1816,16 @@ async fn handle_update_config_inner( .ok_or_else(|| Status::internal("sandbox has no spec"))?; openshell_policy::ensure_sandbox_process_identity(&mut new_policy); + if sandbox_caller { + if openshell_policy::strip_provider_rule_names(&mut new_policy) { + debug!( + sandbox_id = %sandbox_id, + "UpdateConfig: stripped provider-derived policy entries from sandbox sync" + ); + } + } else { + validate_no_reserved_provider_policy_keys(&new_policy)?; + } if let Some(baseline_policy) = spec.policy.as_ref() { validate_static_fields_unchanged(baseline_policy, &new_policy)?; @@ -2270,7 +2282,7 @@ pub(super) async fn handle_submit_policy_analysis( // own rule so the prover sees their contribution honestly. Reject at // the entry boundary — the agent never has reason to address a // provider rule by name. - if chunk.rule_name.starts_with("_provider_") { + if openshell_policy::is_provider_rule_name(&chunk.rule_name) { rejected += 1; rejection_reasons.push(format!( "chunk '{}' uses reserved '_provider_' rule-name prefix", @@ -3459,7 +3471,14 @@ fn parse_proto_add_allow_rules( fn validate_merge_operations_for_server(operations: &[PolicyMergeOp]) -> Result<(), Status> { for operation in operations { match operation { - PolicyMergeOp::AddRule { rule, .. } => validate_rule_not_always_blocked(rule)?, + PolicyMergeOp::AddRule { rule_name, rule } => { + if openshell_policy::is_provider_rule_name(rule_name) { + return Err(Status::invalid_argument(format!( + "merge operation add_rule rule_name '{rule_name}' uses reserved '_provider_' prefix for provider composition" + ))); + } + validate_rule_not_always_blocked(rule)?; + } PolicyMergeOp::AddAllowRules { host, .. } => validate_host_not_always_blocked(host)?, _ => {} } @@ -3577,16 +3596,12 @@ pub(super) async fn merge_chunk_into_policy( ) -> Result<(i64, String), Status> { let rule = NetworkPolicyRule::decode(chunk.proposed_rule.as_slice()) .map_err(|e| Status::internal(format!("decode proposed_rule failed: {e}")))?; - apply_merge_operations_with_retry( - store, - sandbox_id, - None, - &[PolicyMergeOp::AddRule { - rule_name: chunk.rule_name.clone(), - rule, - }], - ) - .await + let operations = [PolicyMergeOp::AddRule { + rule_name: chunk.rule_name.clone(), + rule, + }]; + validate_merge_operations_for_server(&operations)?; + apply_merge_operations_with_retry(store, sandbox_id, None, &operations).await } async fn remove_chunk_from_policy( @@ -4019,6 +4034,19 @@ mod tests { assert!(!is_sandbox_caller(&req)); } + #[test] + fn merge_operation_validation_rejects_reserved_provider_add_rule_name() { + let err = validate_merge_operations_for_server(&[PolicyMergeOp::AddRule { + rule_name: "_provider_work_github".to_string(), + rule: NetworkPolicyRule::default(), + }]) + .unwrap_err(); + + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("_provider_work_github")); + assert!(err.message().contains("reserved '_provider_' prefix")); + } + // ---- Sandbox IDOR guard (issue #1354) ---- #[tokio::test] @@ -4938,7 +4966,7 @@ mod tests { } #[tokio::test] - async fn sandbox_config_preserves_overlapping_user_and_provider_rules() { + async fn sandbox_config_composes_user_and_provider_rules() { let state = test_server_state().await; enable_providers_v2(&state).await; state @@ -4951,7 +4979,7 @@ mod tests { .put_message(&test_sandbox( "sb-overlap", "overlap", - test_policy_with_rule("_provider_work_github", "api.github.com"), + test_policy_with_rule("custom_github", "api.github.com"), vec!["work-github".to_string()], )) .await @@ -4962,17 +4990,17 @@ mod tests { assert!( effective_policy .network_policies - .contains_key("_provider_work_github") + .contains_key("custom_github") ); assert!( effective_policy .network_policies - .contains_key("_provider_work_github_2") + .contains_key("_provider_work_github") ); assert_eq!( effective_policy .network_policies - .get("_provider_work_github") + .get("custom_github") .unwrap() .endpoints[0] .host, @@ -7030,6 +7058,94 @@ mod tests { ); } + #[tokio::test] + async fn approve_draft_chunk_rejects_stored_reserved_provider_rule_name() { + use openshell_core::proto::{NetworkBinary, NetworkEndpoint, NetworkPolicyRule}; + + let state = test_server_state().await; + let sandbox_id = "sb-approve-provider-prefix"; + let sandbox_name = "approve-provider-prefix"; + state + .store + .put_message(&test_sandbox( + sandbox_id, + sandbox_name, + ProtoSandboxPolicy::default(), + vec![], + )) + .await + .unwrap(); + + let proposed_rule = NetworkPolicyRule { + name: "_provider_work_github".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.github.com".to_string(), + port: 443, + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }; + let chunk = DraftChunkRecord { + id: "chunk-provider-prefix".to_string(), + sandbox_id: sandbox_id.to_string(), + draft_version: 1, + status: "pending".to_string(), + rule_name: "_provider_work_github".to_string(), + proposed_rule: proposed_rule.encode_to_vec(), + rationale: "stored legacy/proposal chunk should not approve".to_string(), + security_notes: String::new(), + confidence: 1.0, + created_at_ms: 0, + decided_at_ms: None, + host: "api.github.com".to_string(), + port: 443, + binary: "/usr/bin/curl".to_string(), + hit_count: 1, + first_seen_ms: 0, + last_seen_ms: 0, + validation_result: String::new(), + rejection_reason: String::new(), + }; + state + .store + .put_draft_chunk(&chunk, None) + .await + .expect("draft chunk should persist"); + + let err = handle_approve_draft_chunk( + &state, + with_user(Request::new(ApproveDraftChunkRequest { + name: sandbox_name.to_string(), + chunk_id: chunk.id.clone(), + })), + ) + .await + .expect_err("reserved provider rule names must be rejected at approval"); + + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("_provider_work_github")); + assert!(err.message().contains("reserved '_provider_' prefix")); + let stored_chunk = state + .store + .get_draft_chunk(&chunk.id) + .await + .unwrap() + .expect("chunk should still exist"); + assert_eq!(stored_chunk.status, "pending"); + assert!( + state + .store + .get_latest_policy(sandbox_id) + .await + .unwrap() + .is_none(), + "failed approval must not persist a policy revision" + ); + } + /// v1 calibration row: **L4 with a credential in scope → HIGH finding.** /// The sandbox has a github provider attached, so a credential is in /// scope for api.github.com. A broad L4 proposal therefore lands in @@ -8986,6 +9102,29 @@ mod tests { ); } + #[tokio::test] + async fn update_config_global_policy_rejects_reserved_provider_key() { + let state = test_server_state().await; + + let err = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + global: true, + policy: Some(test_policy_with_rule( + "_provider_work_github", + "api.github.com", + )), + ..Default::default() + })), + ) + .await + .unwrap_err(); + + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("_provider_work_github")); + assert!(err.message().contains("reserved '_provider_' prefix")); + } + #[test] fn merge_effective_settings_global_overrides_sandbox_key() { let global = StoredSettings { @@ -9665,6 +9804,140 @@ mod tests { ); } + #[tokio::test] + async fn update_config_user_policy_rejects_reserved_provider_key() { + let state = test_server_state().await; + state + .store + .put_message(&test_sandbox( + "sb-user-reserved-key", + "user-reserved-key", + test_policy_with_rule("sandbox_only", "sandbox.example.com"), + Vec::new(), + )) + .await + .unwrap(); + + let err = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + name: "user-reserved-key".to_string(), + policy: Some(test_policy_with_rule( + "_provider_work_github", + "api.github.com", + )), + ..Default::default() + })), + ) + .await + .unwrap_err(); + + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("_provider_work_github")); + assert!(err.message().contains("reserved '_provider_' prefix")); + } + + #[tokio::test] + async fn update_config_sandbox_sync_strips_reserved_provider_keys_before_persisting() { + use openshell_core::proto::{SandboxPhase, SandboxSpec}; + + let state = test_server_state().await; + let mut sandbox = Sandbox { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "sb-sync-strip".to_string(), + name: "sync-strip".to_string(), + created_at_ms: 1_000_000, + labels: HashMap::new(), + resource_version: 0, + }), + spec: Some(SandboxSpec { + policy: None, + providers: Vec::new(), + ..Default::default() + }), + ..Default::default() + }; + sandbox.set_phase(SandboxPhase::Provisioning as i32); + state.store.put_message(&sandbox).await.unwrap(); + + let current = state + .store + .get_message_by_name::("sync-strip") + .await + .unwrap() + .unwrap(); + let current_version = current.metadata.as_ref().unwrap().resource_version; + + let mut synced_policy = test_policy_with_rule("sandbox_only", "sandbox.example.com"); + synced_policy.network_policies.insert( + "_provider_work_github".to_string(), + NetworkPolicyRule { + name: "_provider_work_github".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.github.com".to_string(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + + let response = handle_update_config( + &state, + with_sandbox( + Request::new(UpdateConfigRequest { + name: "sync-strip".to_string(), + policy: Some(synced_policy), + expected_resource_version: current_version, + ..Default::default() + }), + "sb-sync-strip", + ), + ) + .await + .unwrap() + .into_inner(); + + assert_eq!(response.version, 1); + + let updated_sandbox = state + .store + .get_message_by_name::("sync-strip") + .await + .unwrap() + .unwrap(); + let spec_policy = updated_sandbox + .spec + .as_ref() + .and_then(|spec| spec.policy.as_ref()) + .expect("spec.policy should be backfilled"); + assert!(spec_policy.network_policies.contains_key("sandbox_only")); + assert!( + !spec_policy + .network_policies + .contains_key("_provider_work_github") + ); + + let persisted = state + .store + .get_latest_policy("sb-sync-strip") + .await + .unwrap() + .expect("policy revision should be persisted"); + let persisted_policy = ProtoSandboxPolicy::decode(persisted.policy_payload.as_slice()) + .expect("persisted policy should decode"); + assert!( + persisted_policy + .network_policies + .contains_key("sandbox_only") + ); + assert!( + !persisted_policy + .network_policies + .contains_key("_provider_work_github") + ); + } + #[tokio::test] async fn update_config_policy_backfill_cas_rejects_stale_version() { use openshell_core::proto::{SandboxPhase, SandboxSpec}; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 28377394f5..1570b8e034 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -46,8 +46,8 @@ use super::provider::{ get_provider_record, is_valid_env_key, validate_provider_environment_keys_unique, }; use super::validation::{ - level_matches, source_matches, validate_exec_request_fields, validate_policy_safety, - validate_sandbox_spec, + level_matches, source_matches, validate_exec_request_fields, + validate_no_reserved_provider_policy_keys, validate_policy_safety, validate_sandbox_spec, }; use super::{MAX_PAGE_SIZE, MAX_PROVIDERS, clamp_limit}; use crate::persistence::current_time_ms; @@ -162,6 +162,7 @@ async fn handle_create_sandbox_inner( // empty, then validate policy safety before persisting. if let Some(ref mut policy) = spec.policy { openshell_policy::ensure_sandbox_process_identity(policy); + validate_no_reserved_provider_policy_keys(policy)?; validate_policy_safety(policy)?; } @@ -2596,6 +2597,37 @@ mod tests { assert!(err.message().contains("provider-b")); } + #[tokio::test] + async fn create_sandbox_rejects_reserved_provider_policy_key() { + let state = test_server_state().await; + let mut policy = openshell_core::proto::SandboxPolicy::default(); + policy.network_policies.insert( + "_provider_work_github".to_string(), + openshell_core::proto::NetworkPolicyRule { + name: "_provider_work_github".to_string(), + ..Default::default() + }, + ); + + let err = handle_create_sandbox( + &state, + Request::new(CreateSandboxRequest { + name: "reserved-policy-key".to_string(), + spec: Some(openshell_core::proto::SandboxSpec { + policy: Some(policy), + ..Default::default() + }), + labels: HashMap::new(), + }), + ) + .await + .unwrap_err(); + + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert!(err.message().contains("_provider_work_github")); + assert!(err.message().contains("reserved '_provider_' prefix")); + } + #[tokio::test] async fn create_sandbox_with_providers_waits_for_sandbox_sync_guard() { let state = test_server_state().await; diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index b3680c6e75..09e9f1cad4 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -638,6 +638,22 @@ pub(super) fn validate_policy_safety(policy: &ProtoSandboxPolicy) -> Result<(), Ok(()) } +/// Validate that user-authored policy does not use provider-derived rule keys. +pub(super) fn validate_no_reserved_provider_policy_keys( + policy: &ProtoSandboxPolicy, +) -> Result<(), Status> { + if let Some(key) = policy + .network_policies + .keys() + .find(|key| openshell_policy::is_provider_rule_name(key)) + { + return Err(Status::invalid_argument(format!( + "network_policies key '{key}' uses reserved '_provider_' prefix for provider composition; use 'openshell policy get --base' for a round-trippable base policy, or use 'openshell policy get --full' to inspect the effective policy including provider entries" + ))); + } + Ok(()) +} + /// Validate that static policy fields (filesystem, landlock, process) haven't changed /// from the baseline (version 1) policy. pub(super) fn validate_static_fields_unchanged( @@ -1605,6 +1621,43 @@ mod tests { assert!(err.message().contains("TLD wildcard")); } + #[test] + fn validate_no_reserved_provider_policy_keys_rejects_reserved_key() { + use openshell_core::proto::NetworkPolicyRule; + + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "_provider_work_github".into(), + NetworkPolicyRule { + name: "_provider_work_github".into(), + ..Default::default() + }, + ); + + let err = validate_no_reserved_provider_policy_keys(&policy).unwrap_err(); + assert_eq!(err.code(), Code::InvalidArgument); + assert!(err.message().contains("_provider_work_github")); + assert!(err.message().contains("reserved '_provider_' prefix")); + assert!(err.message().contains("policy get --base")); + assert!(err.message().contains("round-trippable base policy")); + } + + #[test] + fn validate_no_reserved_provider_policy_keys_accepts_user_key() { + use openshell_core::proto::NetworkPolicyRule; + + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "provider_work_github".into(), + NetworkPolicyRule { + name: "provider_work_github".into(), + ..Default::default() + }, + ); + + assert!(validate_no_reserved_provider_policy_keys(&policy).is_ok()); + } + // ---- Static field validation ---- #[test] diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 406ed12b8d..434ff7c6e0 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -140,13 +140,13 @@ The following steps outline the hot-reload policy update workflow. `--add-allow` and `--add-deny` target existing `protocol: rest` or `protocol: websocket` endpoints. If you pass multiple update flags in one command, OpenShell applies them as one atomic merge batch and persists at most one new revision. -4. For larger edits, pull the current effective policy and edit the YAML directly. Before reusing the file, strip the metadata header above the `---` line. If the sandbox has attached Providers v2 providers, remove generated `_provider_*` entries before reapplying the policy; those entries are derived from provider profiles. +4. For larger edits, pull the current base policy and edit the YAML directly. The base policy is the user-authored policy without provider-composed `_provider_*` entries, so it is safe to round-trip through `openshell policy set`. Before reusing the file, strip the metadata header above the `---` line. ```shell - openshell policy get --full > current-policy.yaml + openshell policy get --base > current-policy.yaml ``` - To inspect a stored sandbox-authored revision instead of the current effective policy, pass `--rev `. + To inspect the effective policy that the sandbox enforces, including provider-composed entries, use `openshell policy get --full`. To inspect a stored sandbox-authored revision instead of the current effective policy, pass `--rev `. 5. Edit the YAML: add or adjust `network_policies` entries, binaries, `access`, or `rules`. diff --git a/docs/sandboxes/providers-v2.mdx b/docs/sandboxes/providers-v2.mdx index 896ac641e1..49e7120fd5 100644 --- a/docs/sandboxes/providers-v2.mdx +++ b/docs/sandboxes/providers-v2.mdx @@ -547,7 +547,7 @@ openshell sandbox create \ -- claude ``` -When `providers_v2_enabled=true`, each attached provider with a matching profile contributes a provider policy layer to the sandbox effective policy. When the setting is disabled, the sandbox receives provider credentials but not provider-derived policy entries. +When `providers_v2_enabled=true`, each attached provider with a matching profile contributes a provider policy layer to the sandbox effective policy. The base policy is the user-authored sandbox policy that you can edit and apply. The effective policy is the composed policy that the sandbox enforces: base policy plus provider policy layers. When the setting is disabled, the sandbox receives provider credentials but not provider-derived policy entries. Updating a custom provider profile affects every provider instance whose `type` matches that profile ID. Provider instances are not rewritten, and sandbox-authored policies are not modified. Running sandboxes observe the updated provider-derived policy on their next config sync. If a gateway-global policy is active, provider-derived policy layers remain suppressed. @@ -565,7 +565,7 @@ The list output includes provider name, provider type, credential key count, and ## Policy Composition -OpenShell stores sandbox-authored policy and provider attachments separately. When a sandbox asks for its effective policy, the gateway composes the current sandbox policy with provider policy layers just in time. +OpenShell stores the base policy and provider attachments separately. When a sandbox asks for its effective policy, the gateway composes the current base policy with provider policy layers just in time. For example, the built-in GitHub profile contains these endpoints and binaries: @@ -643,14 +643,20 @@ network_policies: Inspect the effective policy: ```shell -openshell policy get provider-demo +openshell policy get provider-demo --full +``` + +Pull the round-trippable base policy without provider entries: + +```shell +openshell policy get provider-demo --base ``` Composition follows these rules: - Provider policy entries use reserved `_provider_*` keys derived from provider instance names. -- Provider policy entries are derived data. OpenShell does not persist them back into the sandbox-authored policy. -- If a user-authored rule already uses the same key, OpenShell keeps the user rule and adds a numeric suffix to the provider rule. +- Provider policy entries are derived data. OpenShell does not persist them back into the base policy. +- User-authored policies cannot define `_provider_*` network policy keys. The gateway rejects those keys on sandbox create and full policy replacement, and sandbox-originated policy sync strips them before persistence. - Provider and user rules are concatenated. Overlapping endpoints remain separate rules. - A gateway global policy override suppresses provider-derived policy layers. From 4b78b442e6f6b059439184f4dcad9410ade813b0 Mon Sep 17 00:00:00 2001 From: Calum Murray Date: Fri, 26 Jun 2026 12:17:07 -0400 Subject: [PATCH 13/20] fix(openshell-network-supervisor): gate proxy accept on symlink resolution readiness (#1968) * fix(openshell-network-supervisor): gate proxy accept on symlink resolution readiness Signed-off-by: Calum Murray * fix(openshell-network-supervisor): skip symlink resolution gate when process supervisor is disabled Signed-off-by: Calum Murray --------- Signed-off-by: Calum Murray --- crates/openshell-sandbox/src/lib.rs | 1 + .../openshell-supervisor-network/src/proxy.rs | 25 ++++ .../openshell-supervisor-network/src/run.rs | 136 ++++++++++-------- 3 files changed, 104 insertions(+), 58 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 47550a9715..d5967d1f3e 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -280,6 +280,7 @@ pub async fn run_sandbox( opa_engine.as_ref(), retained_proto.as_ref(), entrypoint_pid.clone(), + process_enabled, &provider_credentials, sandbox_id.as_deref(), sandbox_name_for_agg.as_deref(), diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 691382469e..a5feec9c2d 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -197,6 +197,7 @@ impl ProxyHandle { policy_local_ctx: Option>, denial_tx: Option>, activity_tx: Option, + engine_ready: tokio::sync::watch::Receiver, ) -> Result { // Use override bind_addr, fall back to policy http_addr, then default // to loopback:3128. The default allows the proxy to function when no @@ -238,6 +239,30 @@ impl ProxyHandle { } let join = tokio::spawn(async move { + // Wait for the OPA engine's symlink resolution reload to complete + // before accepting connections. This prevents requests from + // observing a generation transition mid-flight, which would cause + // the generation guard to reject them with a 403. + // + // The TCP listener is already bound, so the OS backlog queues + // incoming SYN packets during this wait. Once we start accepting, + // queued connections drain immediately. + let mut engine_ready = engine_ready; + match tokio::time::timeout( + std::time::Duration::from_secs(15), + engine_ready.wait_for(|v| *v), + ) + .await + { + Ok(_) => {} + Err(_) => { + warn!( + "Engine readiness signal not received within 15s; \ + proceeding with proxy accept loop" + ); + } + } + loop { match listener.accept().await { Ok((stream, _addr)) => { diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index b989230518..9553e06736 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -78,6 +78,7 @@ pub async fn run_networking( opa_engine: Option<&Arc>, retained_proto: Option<&ProtoSandboxPolicy>, entrypoint_pid: Arc, + process_enabled: bool, provider_credentials: &ProviderCredentialState, sandbox_id: Option<&str>, sandbox_name: Option<&str>, @@ -97,80 +98,98 @@ pub async fn run_networking( .or_else(|| sandbox_id.map(str::to_string)), )); + // Readiness signal for the proxy accept loop: the proxy binds the TCP + // listener immediately (so the OS backlog queues early SYN packets) but + // defers `accept()` until symlink resolution completes. This eliminates + // the race where an in-flight request observes a generation transition + // during the OPA engine reload. + let (engine_ready_tx, engine_ready_rx) = tokio::sync::watch::channel(false); + // Spawn a task to resolve policy binary symlinks once the workload's mount // namespace becomes accessible via /proc//root/. The task starts // before run_process spawns the child, so first wait for the orchestrator // to publish a non-zero PID, then poll for proc-root readiness. if let (Some(engine), Some(proto)) = (opa_engine, retained_proto) { - let resolve_engine = engine.clone(); - let resolve_proto = proto.clone(); - let resolve_pid = entrypoint_pid.clone(); - tokio::spawn(async move { - // Phase 1: wait for run_process to publish the entrypoint PID. - // 20 attempts * 250ms = 5s window. - let mut pid = 0; - for attempt in 1..=20 { - pid = resolve_pid.load(Ordering::Acquire); - if pid != 0 { - break; + if process_enabled { + let resolve_engine = engine.clone(); + let resolve_proto = proto.clone(); + let resolve_pid = entrypoint_pid.clone(); + tokio::spawn(async move { + // Phase 1: wait for run_process to publish the entrypoint PID. + // 20 attempts * 250ms = 5s window. + let mut pid = 0; + for attempt in 1..=20 { + pid = resolve_pid.load(Ordering::Acquire); + if pid != 0 { + break; + } + debug!( + attempt, + "Entrypoint PID not yet published, waiting before symlink resolution" + ); + tokio::time::sleep(Duration::from_millis(250)).await; } - debug!( - attempt, - "Entrypoint PID not yet published, waiting before symlink resolution" - ); - tokio::time::sleep(Duration::from_millis(250)).await; - } - if pid == 0 { - warn!( - "Entrypoint PID never published; binary symlink resolution skipped. \ + if pid == 0 { + warn!( + "Entrypoint PID never published; binary symlink resolution skipped. \ Policy binary paths will be matched literally." - ); - return; - } - - // Phase 2: wait for /proc//root/ to become traversable. The - // child's mount namespace is typically ready within a few hundred - // ms of spawn. 10 attempts * 500ms = 5s window. - let probe_path = format!("/proc/{pid}/root/"); - for attempt in 1..=10 { - tokio::time::sleep(Duration::from_millis(500)).await; - if std::fs::metadata(&probe_path).is_ok() { - info!( - pid = pid, - attempt = attempt, - "Container filesystem accessible, resolving policy binary symlinks" ); - match resolve_engine.reload_from_proto_with_pid(&resolve_proto, pid) { - Ok(()) => { - info!( - pid = pid, - "Policy binary symlink resolution complete \ + let _ = engine_ready_tx.send(true); + return; + } + + // Phase 2: wait for /proc//root/ to become traversable. The + // child's mount namespace is typically ready within a few hundred + // ms of spawn. 10 attempts * 500ms = 5s window. + let probe_path = format!("/proc/{pid}/root/"); + for attempt in 1..=10 { + tokio::time::sleep(Duration::from_millis(500)).await; + if std::fs::metadata(&probe_path).is_ok() { + info!( + pid = pid, + attempt = attempt, + "Container filesystem accessible, resolving policy binary symlinks" + ); + match resolve_engine.reload_from_proto_with_pid(&resolve_proto, pid) { + Ok(()) => { + info!( + pid = pid, + "Policy binary symlink resolution complete \ (check logs above for per-binary results)" - ); - } - Err(e) => { - warn!( - "Failed to rebuild OPA engine with symlink resolution \ + ); + } + Err(e) => { + warn!( + "Failed to rebuild OPA engine with symlink resolution \ (non-fatal, falling back to literal path matching): {e}" - ); + ); + } } + let _ = engine_ready_tx.send(true); + return; } - return; + debug!( + pid = pid, + attempt = attempt, + probe_path = %probe_path, + "Container filesystem not yet accessible, retrying symlink resolution" + ); } - debug!( - pid = pid, - attempt = attempt, - probe_path = %probe_path, - "Container filesystem not yet accessible, retrying symlink resolution" - ); - } - warn!( - "Container filesystem /proc/{pid}/root/ not accessible after 10 attempts (5s); \ + warn!( + "Container filesystem /proc/{pid}/root/ not accessible after 10 attempts (5s); \ binary symlink resolution skipped. Policy binary paths will be matched literally. \ If binaries are symlinks, use canonical paths in your policy \ (run 'readlink -f ' inside the sandbox)" - ); - }); + ); + let _ = engine_ready_tx.send(true); + }); + } else { + // No process supervisor — PID will never arrive, skip symlink resolution. + let _ = engine_ready_tx.send(true); + } + } else { + // No symlink resolution needed — unblock the proxy immediately. + let _ = engine_ready_tx.send(true); } // Identity cache for SHA256 TOFU when OPA is active. Only consumed by @@ -279,6 +298,7 @@ pub async fn run_networking( Some(policy_local_ctx.clone()), denial_tx, activity_tx, + engine_ready_rx, ) .await?; Some(proxy_handle) From f569a0ade6cef2e804fe77e86a2816dee64b0f0c Mon Sep 17 00:00:00 2001 From: Jesse Jaggars Date: Fri, 26 Jun 2026 13:52:24 -0400 Subject: [PATCH 14/20] feat(sandbox): proxy-side AWS SigV4 credential signing for CONNECT tunnels (#1638) Signed-off-by: Jesse Jaggars Co-authored-by: Russell Bryant --- Cargo.lock | 250 ++++++-- architecture/sandbox.md | 33 + crates/openshell-policy/src/lib.rs | 232 +++++++ crates/openshell-providers/src/profiles.rs | 12 + .../openshell-supervisor-network/Cargo.toml | 4 + .../src/l7/mod.rs | 97 +++ .../src/l7/relay.rs | 19 + .../src/l7/rest.rs | 377 +++++++++++- .../openshell-supervisor-network/src/lib.rs | 1 + .../openshell-supervisor-network/src/opa.rs | 129 ++++ .../src/policy_local.rs | 3 + .../openshell-supervisor-network/src/proxy.rs | 16 + .../openshell-supervisor-network/src/sigv4.rs | 572 ++++++++++++++++++ .../tests/sigv4_localstack.rs | 239 ++++++++ docs/providers/aws-sigv4.mdx | 151 +++++ docs/reference/policy-schema.mdx | 5 +- proto/sandbox.proto | 10 + 17 files changed, 2113 insertions(+), 37 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/sigv4.rs create mode 100644 crates/openshell-supervisor-network/tests/sigv4_localstack.rs create mode 100644 docs/providers/aws-sigv4.mdx diff --git a/Cargo.lock b/Cargo.lock index 081ffe6be7..672fd71ab3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -267,6 +267,18 @@ dependencies = [ "cc", ] +[[package]] +name = "aws-credential-types" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.16.3" @@ -290,6 +302,112 @@ dependencies = [ "fs_extra", ] +[[package]] +name = "aws-sigv4" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2 0.10.9", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "aws-smithy-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", +] + [[package]] name = "axum" version = "0.8.9" @@ -301,8 +419,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -334,8 +452,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -404,6 +522,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -506,7 +634,7 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", + "http 1.4.0", "http-body-util", "hyper", "hyper-named-pipe", @@ -566,6 +694,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.6.1" @@ -1803,7 +1941,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.0", "indexmap", "slab", "tokio", @@ -1944,6 +2082,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.0" @@ -1963,6 +2112,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1970,7 +2130,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1981,8 +2141,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -2018,8 +2178,8 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2050,7 +2210,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http", + "http 1.4.0", "hyper", "hyper-util", "log", @@ -2085,8 +2245,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "hyper", "ipnet", "libc", @@ -2630,8 +2790,8 @@ dependencies = [ "either", "futures", "home", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -2663,7 +2823,7 @@ checksum = "7845bcc3e0f422df4d9049570baedd9bc1942f0504594e393e72fe24092559cf" dependencies = [ "chrono", "form_urlencoded", - "http", + "http 1.4.0", "json-patch", "k8s-openapi", "schemars", @@ -3276,7 +3436,7 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http", + "http 1.4.0", "rand 0.8.6", "reqwest 0.12.28", "serde", @@ -3305,7 +3465,7 @@ dependencies = [ "bytes", "chrono", "futures-util", - "http", + "http 1.4.0", "http-auth", "jsonwebtoken 10.3.0", "lazy_static", @@ -3680,8 +3840,8 @@ dependencies = [ "glob", "hex", "hmac", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -3752,12 +3912,16 @@ name = "openshell-supervisor-network" version = "0.0.0" dependencies = [ "apollo-parser", + "aws-credential-types", + "aws-sigv4", + "aws-smithy-runtime-api", "base64 0.22.1", "bytes", "flate2", "futures", "glob", "hex", + "http 1.4.0", "ipnet", "libc", "miette", @@ -3876,6 +4040,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owo-colors" version = "4.3.0" @@ -4129,6 +4299,12 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs1" version = "0.7.5" @@ -4689,8 +4865,8 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -4729,8 +4905,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -6314,8 +6490,8 @@ dependencies = [ "base64 0.22.1", "bytes", "h2", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-timeout", @@ -6418,8 +6594,8 @@ dependencies = [ "base64 0.21.7", "bitflags", "bytes", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -6437,8 +6613,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tower 0.5.3", @@ -6562,7 +6738,7 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.0", "httparse", "log", "rand 0.9.4", @@ -6581,7 +6757,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.0", "httparse", "log", "rand 0.9.4", @@ -6769,6 +6945,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -7434,7 +7616,7 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http", + "http 1.4.0", "http-body-util", "hyper", "hyper-util", diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 2552304e16..903af3a4d2 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -90,6 +90,39 @@ and returned access tokens must be bearer-compatible before they are cached or injected. Token response lifetimes are capped and cached with an expiry margin unless a profile supplies an explicit cache TTL override. +For AWS endpoints that require request-level signing, the proxy supports SigV4 +re-signing. When `credential_signing: sigv4` is set on an L7 endpoint, the proxy +strips the client's placeholder-based AWS auth headers, re-signs with real +credentials from the provider, and forwards the request upstream. The signing +mode is auto-detected from the client SDK's `x-amz-content-sha256` header: + +- **Signed body** (hex hash): buffers the request body (up to 10 MiB), computes + its SHA-256, and includes the hash in the signature. Used by Bedrock and most + AWS services. +- **Streaming unsigned** (`STREAMING-UNSIGNED-PAYLOAD-TRAILER`): signs headers + only and streams the body through without buffering. Used by S3 uploads with + `aws-chunked` encoding. +- **Unsigned payload** (`UNSIGNED-PAYLOAD`): signs headers only with no body + hash. Used by S3 over HTTPS for non-chunked requests. + +Chunk-signed streaming modes (`STREAMING-AWS4-HMAC-SHA256-PAYLOAD` and other +`STREAMING-*` variants) are rejected — the proxy cannot reproduce per-chunk +signatures. Use `sigv4:no_body` for those clients. + +Two explicit overrides are available: `credential_signing: sigv4:body` (always +buffer and hash) and `sigv4:no_body` (always unsigned). The `Expect: +100-continue` header is handled within the SigV4 path so clients like boto3 +transmit the body before the proxy forwards to upstream. + +The AWS region is extracted from the endpoint hostname. For non-standard +endpoints (VPC endpoints, custom proxies), set `signing_region` in the policy +endpoint to provide an explicit override. The proxy rejects requests when +neither hostname extraction nor `signing_region` yields a region. + +`credential_signing` and `request_body_credential_rewrite` are mutually +exclusive on the same endpoint. The policy validator rejects policies that +set both. + ## Connect and Logs The supervisor runs an SSH server on a Unix socket inside the sandbox. The diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 22af7093e4..e046954a2d 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -138,6 +138,12 @@ struct NetworkEndpointDef { graphql_persisted_queries: BTreeMap, #[serde(default, skip_serializing_if = "is_zero_u32")] graphql_max_body_bytes: u32, + #[serde(default, skip_serializing_if = "String::is_empty")] + credential_signing: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + signing_service: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + signing_region: String, } // Signature dictated by serde's `skip_serializing_if`, which requires `&T`. @@ -350,6 +356,9 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { }) .collect(), graphql_max_body_bytes: e.graphql_max_body_bytes, + credential_signing: e.credential_signing, + signing_service: e.signing_service, + signing_region: e.signing_region, } }) .collect(), @@ -515,6 +524,9 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { }) .collect(), graphql_max_body_bytes: e.graphql_max_body_bytes, + credential_signing: e.credential_signing.clone(), + signing_service: e.signing_service.clone(), + signing_region: e.signing_region.clone(), } }) .collect(), @@ -697,6 +709,16 @@ pub enum PolicyViolation { TooManyPaths { count: usize }, /// A network endpoint uses a TLD wildcard (e.g. `*.com`). TldWildcard { policy_name: String, host: String }, + /// `credential_signing` is set but `signing_service` is missing. + MissingSigningService { policy_name: String, host: String }, + /// `credential_signing` has an unrecognized value. + UnknownCredentialSigning { + policy_name: String, + host: String, + value: String, + }, + /// `credential_signing` and `request_body_credential_rewrite` are both set. + CredentialSigningWithBodyRewrite { policy_name: String, host: String }, } impl fmt::Display for PolicyViolation { @@ -733,6 +755,31 @@ impl fmt::Display for PolicyViolation { use subdomain wildcards like '*.example.com' instead" ) } + Self::MissingSigningService { policy_name, host } => { + write!( + f, + "network policy '{policy_name}': endpoint '{host}' has credential_signing \ + set but signing_service is empty" + ) + } + Self::UnknownCredentialSigning { + policy_name, + host, + value, + } => { + write!( + f, + "network policy '{policy_name}': endpoint '{host}' has unrecognized \ + credential_signing value '{value}' (expected sigv4, sigv4:body, or sigv4:no_body)" + ) + } + Self::CredentialSigningWithBodyRewrite { policy_name, host } => { + write!( + f, + "network policy '{policy_name}': endpoint '{host}' has both credential_signing \ + and request_body_credential_rewrite set; these options are mutually exclusive" + ) + } } } } @@ -837,6 +884,30 @@ pub fn validate_sandbox_policy( }); } } + if !ep.credential_signing.is_empty() + && !matches!( + ep.credential_signing.as_str(), + "sigv4" | "sigv4:body" | "sigv4:no_body" + ) + { + violations.push(PolicyViolation::UnknownCredentialSigning { + policy_name: name.clone(), + host: ep.host.clone(), + value: ep.credential_signing.clone(), + }); + } + if !ep.credential_signing.is_empty() && ep.signing_service.is_empty() { + violations.push(PolicyViolation::MissingSigningService { + policy_name: name.clone(), + host: ep.host.clone(), + }); + } + if !ep.credential_signing.is_empty() && ep.request_body_credential_rewrite { + violations.push(PolicyViolation::CredentialSigningWithBodyRewrite { + policy_name: name.clone(), + host: ep.host.clone(), + }); + } } } @@ -1380,6 +1451,167 @@ network_policies: assert!(validate_sandbox_policy(&policy).is_ok()); } + #[test] + fn validate_rejects_credential_signing_without_signing_service() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "bedrock".into(), + endpoints: vec![NetworkEndpoint { + host: "bedrock-runtime.us-east-1.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4".into(), + signing_service: String::new(), + ..Default::default() + }], + ..Default::default() + }, + ); + let violations = validate_sandbox_policy(&policy).unwrap_err(); + assert!( + violations + .iter() + .any(|v| matches!(v, PolicyViolation::MissingSigningService { .. })) + ); + } + + #[test] + fn validate_accepts_credential_signing_with_signing_service() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "bedrock".into(), + endpoints: vec![NetworkEndpoint { + host: "bedrock-runtime.us-east-1.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4".into(), + signing_service: "bedrock".into(), + ..Default::default() + }], + ..Default::default() + }, + ); + assert!(validate_sandbox_policy(&policy).is_ok()); + } + + #[test] + fn validate_accepts_sigv4_body_with_signing_service() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "bedrock".into(), + endpoints: vec![NetworkEndpoint { + host: "bedrock-runtime.us-east-1.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4:body".into(), + signing_service: "bedrock".into(), + ..Default::default() + }], + ..Default::default() + }, + ); + assert!(validate_sandbox_policy(&policy).is_ok()); + } + + #[test] + fn validate_accepts_sigv4_no_body_with_signing_service() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "s3".into(), + endpoints: vec![NetworkEndpoint { + host: "s3.us-east-1.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4:no_body".into(), + signing_service: "s3".into(), + ..Default::default() + }], + ..Default::default() + }, + ); + assert!(validate_sandbox_policy(&policy).is_ok()); + } + + #[test] + fn validate_rejects_sigv4_no_body_without_signing_service() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "s3".into(), + endpoints: vec![NetworkEndpoint { + host: "s3.us-east-1.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4:no_body".into(), + signing_service: String::new(), + ..Default::default() + }], + ..Default::default() + }, + ); + let violations = validate_sandbox_policy(&policy).unwrap_err(); + assert!( + violations + .iter() + .any(|v| matches!(v, PolicyViolation::MissingSigningService { .. })) + ); + } + + #[test] + fn validate_rejects_unknown_credential_signing() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "test".into(), + endpoints: vec![NetworkEndpoint { + host: "example.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4_typo".into(), + signing_service: "bedrock".into(), + ..Default::default() + }], + ..Default::default() + }, + ); + let violations = validate_sandbox_policy(&policy).unwrap_err(); + assert!( + violations + .iter() + .any(|v| matches!(v, PolicyViolation::UnknownCredentialSigning { .. })) + ); + } + + #[test] + fn validate_rejects_credential_signing_with_body_rewrite() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "aws".into(), + NetworkPolicyRule { + name: "bedrock".into(), + endpoints: vec![NetworkEndpoint { + host: "bedrock-runtime.us-east-1.amazonaws.com".into(), + port: 443, + credential_signing: "sigv4".into(), + signing_service: "bedrock".into(), + request_body_credential_rewrite: true, + ..Default::default() + }], + ..Default::default() + }, + ); + let violations = validate_sandbox_policy(&policy).unwrap_err(); + assert!( + violations + .iter() + .any(|v| matches!(v, PolicyViolation::CredentialSigningWithBodyRewrite { .. })) + ); + } + #[test] fn normalize_path_collapses_separators() { assert_eq!(normalize_path("/usr//lib"), "/usr/lib"); diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index fe647d523a..530318fb72 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -205,6 +205,12 @@ pub struct EndpointProfile { pub graphql_max_body_bytes: u32, #[serde(default, skip_serializing_if = "String::is_empty")] pub path: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub credential_signing: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub signing_service: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub signing_region: String, } #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] @@ -775,6 +781,9 @@ fn endpoint_to_proto(endpoint: &EndpointProfile) -> NetworkEndpoint { .collect(), graphql_max_body_bytes: endpoint.graphql_max_body_bytes, path: endpoint.path.clone(), + credential_signing: endpoint.credential_signing.clone(), + signing_service: endpoint.signing_service.clone(), + signing_region: endpoint.signing_region.clone(), } } @@ -805,6 +814,9 @@ fn endpoint_from_proto(endpoint: &NetworkEndpoint) -> EndpointProfile { .collect(), graphql_max_body_bytes: endpoint.graphql_max_body_bytes, path: endpoint.path.clone(), + credential_signing: endpoint.credential_signing.clone(), + signing_service: endpoint.signing_service.clone(), + signing_region: endpoint.signing_region.clone(), } } diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 0f19d1ff30..57d1ef4bb8 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -17,6 +17,10 @@ openshell-policy = { path = "../openshell-policy" } openshell-router = { path = "../openshell-router" } apollo-parser = { workspace = true } +aws-sigv4 = { version = "1", features = ["sign-http", "http1"] } +aws-credential-types = { version = "1", features = ["hardcoded-credentials"] } +aws-smithy-runtime-api = { version = "1", features = ["client"] } +http = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } flate2 = "1" diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index 802058ec25..b0f52a3ffb 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -51,6 +51,26 @@ pub enum TlsMode { Skip, } +/// Credential signing mode for proxy-side request signing. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum CredentialSigning { + #[default] + None, + /// Auto-detect: include body in signature when Content-Length is present, + /// skip body when Transfer-Encoding is chunked or body is absent. + SigV4, + /// Always include body in signature (buffer body, compute SHA-256 hash). + SigV4Body, + /// Never include body in signature (use UNSIGNED-PAYLOAD, stream through). + SigV4NoBody, +} + +impl CredentialSigning { + pub fn is_sigv4(&self) -> bool { + matches!(self, Self::SigV4 | Self::SigV4Body | Self::SigV4NoBody) + } +} + /// Enforcement mode for L7 policy decisions. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum EnforcementMode { @@ -89,6 +109,14 @@ pub struct L7EndpointConfig { /// When true, client-to-server GraphQL-over-WebSocket operation messages /// are classified with the same operation policy used by GraphQL-over-HTTP. pub websocket_graphql_policy: bool, + /// Proxy-side credential signing mode for this endpoint. + pub credential_signing: CredentialSigning, + /// AWS signing service name (e.g. `"bedrock"`). Required when + /// `credential_signing` is `SigV4`. + pub signing_service: String, + /// AWS region override for `SigV4` signing. When set, takes precedence + /// over hostname-based region extraction. + pub signing_region: String, } /// Result of an L7 policy decision for a single request. @@ -166,6 +194,37 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { .filter(|v| *v > 0) .unwrap_or(graphql::DEFAULT_MAX_BODY_BYTES); + let credential_signing = match get_object_str(val, "credential_signing").as_deref() { + Some("sigv4") => CredentialSigning::SigV4, + Some("sigv4:body") => CredentialSigning::SigV4Body, + Some("sigv4:no_body") => CredentialSigning::SigV4NoBody, + Some(other) if !other.is_empty() => { + let event = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Other) + .severity(openshell_ocsf::SeverityId::High) + .message(format!( + "rejecting endpoint: unrecognized credential_signing value {other:?}" + )) + .build(); + openshell_ocsf::ocsf_emit!(event); + return None; + } + _ => CredentialSigning::None, + }; + + let signing_service = get_object_str(val, "signing_service").unwrap_or_default(); + let signing_region = get_object_str(val, "signing_region").unwrap_or_default(); + + if credential_signing.is_sigv4() && signing_service.is_empty() { + let event = openshell_ocsf::NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(openshell_ocsf::ActivityId::Other) + .severity(openshell_ocsf::SeverityId::High) + .message("rejecting endpoint: credential_signing requires signing_service".to_string()) + .build(); + openshell_ocsf::ocsf_emit!(event); + return None; + } + Some(L7EndpointConfig { protocol, path: get_object_str(val, "path").unwrap_or_default(), @@ -176,6 +235,9 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { websocket_credential_rewrite, request_body_credential_rewrite, websocket_graphql_policy, + credential_signing, + signing_service, + signing_region, }) } @@ -1197,6 +1259,41 @@ mod tests { assert_eq!(config.enforcement, EnforcementMode::Audit); } + #[test] + fn parse_credential_signing_sigv4() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "rest", "credential_signing": "sigv4", "signing_service": "bedrock", "host": "bedrock.us-east-1.amazonaws.com", "port": 443}"#, + ).unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert_eq!(config.credential_signing, CredentialSigning::SigV4); + assert!(config.credential_signing.is_sigv4()); + } + + #[test] + fn parse_credential_signing_sigv4_body() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "rest", "credential_signing": "sigv4:body", "signing_service": "bedrock", "host": "bedrock.us-east-1.amazonaws.com", "port": 443}"#, + ).unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert_eq!(config.credential_signing, CredentialSigning::SigV4Body); + assert!(config.credential_signing.is_sigv4()); + } + + #[test] + fn parse_credential_signing_sigv4_no_body() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "rest", "credential_signing": "sigv4:no_body", "signing_service": "s3", "host": "s3.us-east-1.amazonaws.com", "port": 443}"#, + ).unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert_eq!(config.credential_signing, CredentialSigning::SigV4NoBody); + assert!(config.credential_signing.is_sigv4()); + } + + #[test] + fn is_sigv4_false_for_none() { + assert!(!CredentialSigning::None.is_sigv4()); + } + #[test] fn parse_l7_config_websocket_protocol() { let val = regorus::Value::from_json_str( diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 830e3461b2..5c95760a1d 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -407,6 +407,11 @@ where websocket_extensions: websocket_extension_mode(config), request_body_credential_rewrite: config.protocol == L7Protocol::Rest && config.request_body_credential_rewrite, + credential_signing: config.credential_signing, + signing_service: &config.signing_service, + signing_region: &config.signing_region, + host: &ctx.host, + port: ctx.port, }, ) .await?; @@ -851,6 +856,11 @@ where websocket_extensions: websocket_extension_mode(config), request_body_credential_rewrite: config.protocol == L7Protocol::Rest && config.request_body_credential_rewrite, + credential_signing: config.credential_signing, + signing_service: &config.signing_service, + signing_region: &config.signing_region, + host: &ctx.host, + port: ctx.port, }, ) .await?; @@ -1881,6 +1891,9 @@ network_policies: websocket_credential_rewrite: true, request_body_credential_rewrite: false, websocket_graphql_policy: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), }]; let ctx = L7EvalContext { host: "gateway.example.test".into(), @@ -1984,6 +1997,9 @@ network_policies: websocket_credential_rewrite: true, request_body_credential_rewrite: false, websocket_graphql_policy: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), }]; let (child_env, resolver) = SecretResolver::from_provider_env( std::iter::once(("DISCORD_BOT_TOKEN".to_string(), "real-token".to_string())).collect(), @@ -2104,6 +2120,9 @@ network_policies: websocket_credential_rewrite: true, request_body_credential_rewrite: false, websocket_graphql_policy: true, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), }]; let (child_env, resolver) = SecretResolver::from_provider_env( std::iter::once(("T".to_string(), "real-token".to_string())).collect(), diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 3ae46417f3..65fd09fb6f 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -9,18 +9,24 @@ use crate::l7::provider::{BodyLength, L7Provider, L7Request, RelayOutcome}; use crate::opa::PolicyGenerationGuard; +use aws_sigv4::http_request::SignableBody; use base64::Engine as _; use miette::{IntoDiagnostic, Result, miette}; use openshell_core::secrets::{ SecretResolver, contains_reserved_credential_marker, rewrite_http_header_block, }; +use openshell_ocsf::ctx::ctx as ocsf_ctx; use sha1::{Digest, Sha1}; use std::collections::{HashMap, HashSet}; +use std::fmt; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tracing::debug; const MAX_HEADER_BYTES: usize = 16384; // 16 KiB for HTTP headers const MAX_REWRITE_BODY_BYTES: usize = 256 * 1024; +/// Maximum body bytes for `SigV4` body-signing mode. Larger than the credential +/// rewrite limit because Bedrock payloads can be several megabytes. +const MAX_SIGV4_BODY_BYTES: usize = 10 * 1024 * 1024; const RELAY_BUF_SIZE: usize = 8192; const HTTP_METHOD_PREFIXES: &[&[u8]] = &[ b"GET ", @@ -393,6 +399,11 @@ where generation_guard, websocket_extensions: WebSocketExtensionMode::Preserve, request_body_credential_rewrite: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: "", + signing_region: "", + host: "", + port: 0, }, ) .await @@ -411,6 +422,11 @@ pub(crate) struct RelayRequestOptions<'a> { pub(crate) generation_guard: Option<&'a PolicyGenerationGuard>, pub(crate) websocket_extensions: WebSocketExtensionMode, pub(crate) request_body_credential_rewrite: bool, + pub(crate) credential_signing: crate::l7::CredentialSigning, + pub(crate) signing_service: &'a str, + pub(crate) signing_region: &'a str, + pub(crate) host: &'a str, + pub(crate) port: u16, } pub(crate) async fn relay_http_request_with_options_guarded( @@ -437,8 +453,19 @@ where parse_websocket_upgrade_request(&req.raw_header[..header_end])? }; + // When SigV4 signing is configured, strip AWS auth headers before credential + // rewriting so the fail-closed placeholder scan doesn't reject the SigV4 + // Authorization header (which embeds placeholder strings). + let raw_for_rewrite; + let header_source = if options.credential_signing.is_sigv4() { + raw_for_rewrite = crate::sigv4::strip_aws_headers(&req.raw_header[..header_end])?; + &raw_for_rewrite[..] + } else { + &req.raw_header[..header_end] + }; + let (header_bytes, expected_websocket_extension) = rewrite_websocket_extensions_for_mode( - &req.raw_header[..header_end], + header_source, options.websocket_extensions, websocket_request.is_some(), )?; @@ -458,7 +485,219 @@ where guard.ensure_current()?; } - if options.request_body_credential_rewrite { + // Apply SigV4 signing if configured. + if options.credential_signing.is_sigv4() { + // Defense-in-depth: credential_signing and request_body_credential_rewrite + // are mutually exclusive (validated at policy load time). + if options.request_body_credential_rewrite { + return Err(miette!( + "credential_signing and request_body_credential_rewrite are \ + mutually exclusive on the same endpoint" + )); + } + // SigV4 re-signing needs the body before forwarding. If the client + // sent `Expect: 100-continue`, acknowledge it so the client transmits + // the body. Scoped to SigV4 paths only — non-SigV4 traffic forwards + // the Expect header to upstream for normal handling. + if has_expect_continue(header_str) { + client + .write_all(b"HTTP/1.1 100 Continue\r\n\r\n") + .await + .into_diagnostic()?; + client.flush().await.into_diagnostic()?; + } + if let Some(resolver) = options.resolver { + let access_key_placeholder = + openshell_core::secrets::placeholder_for_env_key("AWS_ACCESS_KEY_ID"); + let secret_key_placeholder = + openshell_core::secrets::placeholder_for_env_key("AWS_SECRET_ACCESS_KEY"); + let session_token_placeholder = + openshell_core::secrets::placeholder_for_env_key("AWS_SESSION_TOKEN"); + + match ( + resolver.resolve_placeholder(&access_key_placeholder), + resolver.resolve_placeholder(&secret_key_placeholder), + ) { + (Some(access_key), Some(secret_key)) => { + let session_token = resolver.resolve_placeholder(&session_token_placeholder); + // Use explicit signing_region from policy if set, + // otherwise extract from hostname. + let region = if options.signing_region.is_empty() { + match crate::sigv4::extract_aws_region(options.host) { + Some(r) => r, + None => { + return Err(miette!( + "SigV4 signing: cannot extract AWS region from \ + hostname '{host}'; set signing_region in the \ + policy endpoint", + host = options.host, + )); + } + } + } else { + options.signing_region.to_string() + }; + let service = &options.signing_service; + if service.is_empty() { + return Err(miette!( + "SigV4 signing configured but signing_service not set in policy" + )); + } + + let payload_mode = match options.credential_signing { + crate::l7::CredentialSigning::SigV4Body => SigV4PayloadMode::SignBody, + crate::l7::CredentialSigning::SigV4NoBody => { + SigV4PayloadMode::UnsignedPayload + } + crate::l7::CredentialSigning::SigV4 => detect_payload_mode(header_str)?, + crate::l7::CredentialSigning::None => unreachable!(), + }; + + if payload_mode == SigV4PayloadMode::SignBody { + // Buffer body and include its hash in the signature. + // This requires Content-Length — chunked bodies cannot + // be buffered for signing. detect_payload_mode() should + // route chunked requests to the streaming path, but + // guard here as defense-in-depth. + let body_length = parse_body_length(header_str)?; + if matches!(body_length, BodyLength::Chunked) { + return Err(miette!( + "SigV4 body signing requires Content-Length; \ + chunked transfer encoding is not supported in this mode" + )); + } + // NOTE(defense-in-depth): Build the full request from + // rewritten headers + body. `rewrite_result.rewritten` + // has already had AWS auth headers stripped by + // `strip_aws_headers`; `apply_sigv4_to_request` strips + // them again internally via `parse_request_parts` — + // the redundancy is intentional. + let overflow = &req.raw_header[header_end..]; + let mut full_request = rewrite_result.rewritten.clone(); + full_request.extend_from_slice(overflow); + if let BodyLength::ContentLength(body_len) = body_length { + if body_len > MAX_SIGV4_BODY_BYTES as u64 { + return Err(miette!( + "SigV4 body signing buffers at most {MAX_SIGV4_BODY_BYTES} bytes" + )); + } + let already_have = overflow.len() as u64; + if body_len > already_have { + let remaining = + usize::try_from(body_len - already_have).unwrap_or(usize::MAX); + let mut body_buf = vec![0u8; remaining]; + client.read_exact(&mut body_buf).await.into_diagnostic()?; + full_request.extend_from_slice(&body_buf); + } + } + + // Re-check policy after body buffering — a slow upload + // may have outlived a policy reload. + if let Some(guard) = options.generation_guard { + guard.ensure_current()?; + } + + let signed = crate::sigv4::apply_sigv4_to_request( + &full_request, + options.host, + ®ion, + service, + access_key, + secret_key, + session_token, + )?; + upstream.write_all(&signed).await.into_diagnostic()?; + } else { + // Sign headers only, stream body through. + let signable_body = match payload_mode { + SigV4PayloadMode::StreamingUnsignedTrailer => { + SignableBody::StreamingUnsignedPayloadTrailer + } + _ => SignableBody::UnsignedPayload, + }; + let signed_headers = crate::sigv4::apply_sigv4_headers_only_with_body( + &rewrite_result.rewritten, + options.host, + ®ion, + service, + access_key, + secret_key, + session_token, + signable_body, + )?; + upstream + .write_all(&signed_headers) + .await + .into_diagnostic()?; + + let overflow = &req.raw_header[header_end..]; + if !overflow.is_empty() { + if let Some(guard) = options.generation_guard { + guard.ensure_current()?; + } + upstream.write_all(overflow).await.into_diagnostic()?; + } + let overflow_len = overflow.len() as u64; + + match req.body_length { + BodyLength::ContentLength(len) => { + let remaining = len.saturating_sub(overflow_len); + if remaining > 0 { + relay_fixed( + client, + upstream, + remaining, + options.generation_guard, + ) + .await?; + } + } + BodyLength::Chunked => { + relay_chunked( + client, + upstream, + &req.raw_header[header_end..], + options.generation_guard, + ) + .await?; + } + BodyLength::None => {} + } + } + + // OCSF event after successful signing and upstream write. + let event = openshell_ocsf::NetworkActivityBuilder::new( + ocsf_ctx(), + ) + .activity(openshell_ocsf::ActivityId::Traffic) + .action(openshell_ocsf::ActionId::Allowed) + .disposition(openshell_ocsf::DispositionId::Allowed) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .dst_endpoint(openshell_ocsf::Endpoint::from_domain( + options.host, + options.port, + )) + .message(format!( + "SigV4 re-signed {host}:{port} service={service} region={region} mode={payload_mode}", + host = options.host, + port = options.port, + )) + .build(); + openshell_ocsf::ocsf_emit!(event); + } + _ => { + return Err(miette!( + "SigV4 signing configured but AWS credentials not found in provider" + )); + } + } + } else { + return Err(miette!( + "SigV4 signing configured but no secret resolver available" + )); + } + } else if options.request_body_credential_rewrite { let body = collect_and_rewrite_request_body( req, client, @@ -1374,6 +1613,76 @@ fn non_empty(value: Option<&str>) -> Option<&str> { value.map(str::trim).filter(|value| !value.is_empty()) } +/// Check if the request includes `Expect: 100-continue`. +fn has_expect_continue(headers: &str) -> bool { + headers.lines().skip(1).any(|line| { + let lower = line.to_ascii_lowercase(); + lower.starts_with("expect:") + && lower + .split_once(':') + .is_some_and(|(_, v)| v.trim() == "100-continue") + }) +} + +/// Resolved payload signing mode for a `SigV4` request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SigV4PayloadMode { + /// Buffer body and include its SHA-256 hash in the signature. + SignBody, + /// Use literal `UNSIGNED-PAYLOAD` — no body buffering needed. + UnsignedPayload, + /// Use `STREAMING-UNSIGNED-PAYLOAD-TRAILER` for `aws-chunked` streams. + StreamingUnsignedTrailer, +} + +impl fmt::Display for SigV4PayloadMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::SignBody => write!(f, "sign_body"), + Self::UnsignedPayload => write!(f, "unsigned_payload"), + Self::StreamingUnsignedTrailer => write!(f, "streaming_unsigned_trailer"), + } + } +} + +/// Auto-detect the payload signing mode from the client's original headers. +/// +/// Mirrors the mode the client SDK chose by inspecting `x-amz-content-sha256`: +/// - `STREAMING-UNSIGNED-PAYLOAD-TRAILER` → `StreamingUnsignedTrailer` +/// - `UNSIGNED-PAYLOAD` → `UnsignedPayload` +/// - Hex hash → `SignBody` (buffer + hash, requires `Content-Length`) +/// - `STREAMING-AWS4-HMAC-SHA256-PAYLOAD` → **rejected** (the proxy cannot +/// reproduce per-chunk signatures; use `sigv4:no_body` instead) +/// - Other `STREAMING-*` values → **rejected** (unsupported streaming mode) +/// - Absent → `SignBody` if `Content-Length` present, else `UnsignedPayload` +fn detect_payload_mode(headers: &str) -> Result { + for line in headers.lines().skip(1) { + let lower = line.to_ascii_lowercase(); + if lower.starts_with("x-amz-content-sha256:") { + let val = lower.split_once(':').map_or("", |(_, v)| v.trim()); + return match val { + "streaming-unsigned-payload-trailer" => { + Ok(SigV4PayloadMode::StreamingUnsignedTrailer) + } + "unsigned-payload" => Ok(SigV4PayloadMode::UnsignedPayload), + v if v.starts_with("streaming-") => Err(miette!( + "SigV4 auto-detect does not support chunk-signed streaming mode \ + '{v}'; use credential_signing: sigv4:no_body to stream \ + with UNSIGNED-PAYLOAD instead" + )), + _ => Ok(SigV4PayloadMode::SignBody), + }; + } + } + Ok( + if matches!(parse_body_length(headers)?, BodyLength::ContentLength(_)) { + SigV4PayloadMode::SignBody + } else { + SigV4PayloadMode::UnsignedPayload + }, + ) +} + /// Parse Content-Length or Transfer-Encoding from HTTP headers. /// /// Per RFC 7230 Section 3.3.3, rejects requests containing both @@ -5099,4 +5408,68 @@ mod tests { "Relay should fail when path placeholder cannot be resolved" ); } + + #[test] + fn detect_payload_mode_unsigned_payload() { + let headers = "PUT /bucket/key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nX-Amz-Content-Sha256: UNSIGNED-PAYLOAD\r\n\r\n"; + assert_eq!( + detect_payload_mode(headers).unwrap(), + SigV4PayloadMode::UnsignedPayload + ); + } + + #[test] + fn detect_payload_mode_streaming_unsigned_trailer() { + let headers = "PUT /bucket/key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nX-Amz-Content-Sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER\r\n\r\n"; + assert_eq!( + detect_payload_mode(headers).unwrap(), + SigV4PayloadMode::StreamingUnsignedTrailer + ); + } + + #[test] + fn detect_payload_mode_hex_hash_is_sign_body() { + let headers = "POST /model/invoke HTTP/1.1\r\nHost: bedrock.amazonaws.com\r\nX-Amz-Content-Sha256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\r\nContent-Length: 10\r\n\r\n"; + assert_eq!( + detect_payload_mode(headers).unwrap(), + SigV4PayloadMode::SignBody + ); + } + + #[test] + fn detect_payload_mode_rejects_chunk_signed_streaming() { + let headers = "PUT /bucket/key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nX-Amz-Content-Sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD\r\n\r\n"; + let result = detect_payload_mode(headers); + assert!(result.is_err()); + let msg = result.unwrap_err().to_string(); + assert!( + msg.contains("sigv4:no_body"), + "error should suggest sigv4:no_body, got: {msg}" + ); + } + + #[test] + fn detect_payload_mode_rejects_unknown_streaming() { + let headers = "PUT /bucket/key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nX-Amz-Content-Sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER\r\n\r\n"; + let result = detect_payload_mode(headers); + assert!(result.is_err()); + } + + #[test] + fn detect_payload_mode_absent_with_content_length() { + let headers = "POST /model/invoke HTTP/1.1\r\nHost: bedrock.amazonaws.com\r\nContent-Length: 42\r\n\r\n"; + assert_eq!( + detect_payload_mode(headers).unwrap(), + SigV4PayloadMode::SignBody + ); + } + + #[test] + fn detect_payload_mode_absent_without_content_length() { + let headers = "GET /bucket HTTP/1.1\r\nHost: s3.amazonaws.com\r\n\r\n"; + assert_eq!( + detect_payload_mode(headers).unwrap(), + SigV4PayloadMode::UnsignedPayload + ); + } } diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index a559a57e6e..c6c1af5aed 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -16,5 +16,6 @@ pub mod policy_local; pub mod procfs; pub mod proxy; pub mod run; +pub mod sigv4; mod spiffe_endpoint; mod token_grant; diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 4dd0350ff2..0e97f58571 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -1118,6 +1118,15 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St if e.request_body_credential_rewrite { ep["request_body_credential_rewrite"] = true.into(); } + if !e.credential_signing.is_empty() { + ep["credential_signing"] = e.credential_signing.clone().into(); + } + if !e.signing_service.is_empty() { + ep["signing_service"] = e.signing_service.clone().into(); + } + if !e.signing_region.is_empty() { + ep["signing_region"] = e.signing_region.clone().into(); + } if !e.persisted_queries.is_empty() { ep["persisted_queries"] = e.persisted_queries.clone().into(); } @@ -2720,6 +2729,126 @@ network_policies: assert!(l7.websocket_credential_rewrite); } + #[test] + fn l7_endpoint_config_preserves_proto_credential_signing() { + let mut network_policies = std::collections::HashMap::new(); + network_policies.insert( + "bedrock".to_string(), + NetworkPolicyRule { + name: "bedrock".to_string(), + endpoints: vec![NetworkEndpoint { + host: "bedrock-runtime.us-east-2.amazonaws.com".to_string(), + port: 443, + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + access: "read-write".to_string(), + credential_signing: "sigv4".to_string(), + signing_service: "bedrock".to_string(), + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/local/bin/claude".to_string(), + ..Default::default() + }], + }, + ); + let proto = ProtoSandboxPolicy { + version: 1, + filesystem: Some(ProtoFs { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }), + landlock: Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".to_string(), + }), + process: Some(ProtoProc { + run_as_user: "sandbox".to_string(), + run_as_group: "sandbox".to_string(), + }), + network_policies, + }; + + let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); + let input = NetworkInput { + host: "bedrock-runtime.us-east-2.amazonaws.com".into(), + port: 443, + binary_path: PathBuf::from("/usr/local/bin/claude"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + let config = engine + .query_endpoint_config(&input) + .unwrap() + .expect("endpoint config"); + let l7 = crate::l7::parse_l7_config(&config).unwrap(); + assert_eq!(l7.credential_signing, crate::l7::CredentialSigning::SigV4); + assert_eq!(l7.signing_service, "bedrock"); + } + + #[test] + fn l7_endpoint_config_preserves_proto_signing_region() { + let mut network_policies = std::collections::HashMap::new(); + network_policies.insert( + "custom_vpc".to_string(), + NetworkPolicyRule { + name: "custom_vpc".to_string(), + endpoints: vec![NetworkEndpoint { + host: "custom-vpc-endpoint.example.com".to_string(), + port: 443, + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + access: "full".to_string(), + credential_signing: "sigv4".to_string(), + signing_service: "s3".to_string(), + signing_region: "us-west-2".to_string(), + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/local/bin/aws".to_string(), + ..Default::default() + }], + }, + ); + let proto = ProtoSandboxPolicy { + version: 1, + filesystem: Some(ProtoFs { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }), + landlock: Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".to_string(), + }), + process: Some(ProtoProc { + run_as_user: "sandbox".to_string(), + run_as_group: "sandbox".to_string(), + }), + network_policies, + }; + + let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); + let input = NetworkInput { + host: "custom-vpc-endpoint.example.com".into(), + port: 443, + binary_path: PathBuf::from("/usr/local/bin/aws"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + let config = engine + .query_endpoint_config(&input) + .unwrap() + .expect("endpoint config"); + let l7 = crate::l7::parse_l7_config(&config).unwrap(); + assert_eq!(l7.credential_signing, crate::l7::CredentialSigning::SigV4); + assert_eq!(l7.signing_service, "s3"); + assert_eq!(l7.signing_region, "us-west-2"); + } + #[test] fn l7_endpoint_config_preserves_proto_request_body_credential_rewrite() { let mut network_policies = std::collections::HashMap::new(); diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index 2fce25389a..b3b6842479 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -1126,6 +1126,9 @@ fn network_endpoint_from_json( graphql_persisted_queries: HashMap::new(), graphql_max_body_bytes: 0, path: String::new(), + credential_signing: String::new(), + signing_service: String::new(), + signing_region: String::new(), }) } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index a5feec9c2d..bba3c3919a 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -284,6 +284,7 @@ impl ProxyHandle { let dtx = denial_tx.clone(); let atx = activity_tx.clone(); tokio::spawn(async move { + #[allow(clippy::large_futures)] if let Err(err) = handle_tcp_connection( stream, opa, @@ -3072,6 +3073,11 @@ where generation_guard: Some(options.generation_guard), websocket_extensions: options.websocket_extensions, request_body_credential_rewrite: options.request_body_credential_rewrite, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: "", + signing_region: "", + host: "", + port: 0, }, ) .await @@ -4140,6 +4146,7 @@ async fn handle_forward_proxy( return Ok(()); } }; + if let Err(e) = forward_generation_guard.ensure_current() { emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); respond( @@ -4291,6 +4298,9 @@ mod tests { websocket_credential_rewrite, request_body_credential_rewrite: false, websocket_graphql_policy: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), } } @@ -5007,6 +5017,9 @@ network_policies: websocket_credential_rewrite: false, request_body_credential_rewrite: false, websocket_graphql_policy: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), }, }, L7ConfigSnapshot { @@ -5020,6 +5033,9 @@ network_policies: websocket_credential_rewrite: false, request_body_credential_rewrite: false, websocket_graphql_policy: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), }, }, ]; diff --git a/crates/openshell-supervisor-network/src/sigv4.rs b/crates/openshell-supervisor-network/src/sigv4.rs new file mode 100644 index 0000000000..40cc189dc2 --- /dev/null +++ b/crates/openshell-supervisor-network/src/sigv4.rs @@ -0,0 +1,572 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use aws_credential_types::Credentials; +use aws_sigv4::http_request::{ + PayloadChecksumKind, SignableBody, SignableRequest, SigningSettings, sign, +}; +use aws_sigv4::sign::v4; +use aws_smithy_runtime_api::client::identity::Identity; +use miette::{Result, miette}; +use std::time::SystemTime; + +/// AWS regions contain a hyphen followed by a digit (e.g., `us-east-1`). +/// Service names like `s3` or `bedrock-runtime` do not. +fn looks_like_region(s: &str) -> bool { + let bytes = s.as_bytes(); + for i in 0..bytes.len().saturating_sub(1) { + if bytes[i] == b'-' && bytes[i + 1].is_ascii_digit() { + return true; + } + } + false +} + +/// Extract the AWS region from an AWS hostname. +/// +/// Supports standard, dualstack, FIPS, virtual-hosted, and China partition +/// hostnames. The region is the label immediately before `amazonaws.com` +/// (or `amazonaws.com.cn`). +pub fn extract_aws_region(host: &str) -> Option { + let parts: Vec<&str> = host.split('.').collect(); + // China partition: *.amazonaws.com.cn + if parts.len() >= 5 + && parts[parts.len() - 3] == "amazonaws" + && parts[parts.len() - 2] == "com" + && parts[parts.len() - 1] == "cn" + { + let candidate = parts[parts.len() - 4]; + if looks_like_region(candidate) { + return Some(candidate.to_string()); + } + return None; + } + // Standard/dualstack/FIPS/virtual-hosted: *.amazonaws.com + // Scan right-to-left from "amazonaws", skipping non-region labels + // like "dualstack". Handles: s3.us-east-1.amazonaws.com, + // s3.dualstack.us-west-2.amazonaws.com, + // s3-fips.dualstack.us-west-2.amazonaws.com, etc. + if parts.len() >= 4 && parts[parts.len() - 2] == "amazonaws" && parts[parts.len() - 1] == "com" + { + let mut idx = parts.len() - 3; + while idx > 0 && parts[idx] == "dualstack" { + idx -= 1; + } + if idx > 0 && looks_like_region(parts[idx]) { + return Some(parts[idx].to_string()); + } + } + None +} + +/// Strip AWS auth headers from raw HTTP request bytes. +/// +/// Removes `Authorization`, `X-Amz-Date`, `X-Amz-Security-Token`, and +/// `X-Amz-Content-Sha256` headers so the request can pass through the +/// proxy's fail-closed placeholder scan before re-signing. +/// +/// Returns `Err` if the header block is not valid UTF-8. Failing closed +/// prevents non-UTF-8 requests from passing through with their original +/// AWS credentials intact. +pub fn strip_aws_headers(raw: &[u8]) -> Result> { + let header_end = raw + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(raw.len(), |p| p + 4); + + // The caller (rest.rs) validates UTF-8 strictly before reaching this + // point, so `from_utf8` should never fail here. Fail closed so that + // crafted non-UTF-8 bytes cannot smuggle AWS credentials through. + let header_str = std::str::from_utf8(&raw[..header_end]) + .map_err(|e| miette!("strip_aws_headers: header block is not valid UTF-8: {e}"))?; + let lines: Vec<&str> = header_str.split("\r\n").collect(); + + let mut output = Vec::with_capacity(raw.len()); + + for (i, line) in lines.iter().enumerate() { + if i == 0 { + output.extend_from_slice(line.as_bytes()); + output.extend_from_slice(b"\r\n"); + continue; + } + if line.is_empty() { + break; + } + let lower = line.to_ascii_lowercase(); + if lower.starts_with("authorization:") + || lower.starts_with("x-amz-date:") + || lower.starts_with("x-amz-security-token:") + || lower.starts_with("x-amz-content-sha256:") + { + continue; + } + output.extend_from_slice(line.as_bytes()); + output.extend_from_slice(b"\r\n"); + } + + output.extend_from_slice(b"\r\n"); + + if header_end < raw.len() { + output.extend_from_slice(&raw[header_end..]); + } + + Ok(output) +} + +struct RequestParts<'a> { + method: &'a str, + path: &'a str, + request_line: &'a str, + headers_to_sign: Vec<(String, String)>, + all_headers: Vec<(String, String)>, +} + +/// Parse raw HTTP headers into components needed for `SigV4` signing. +/// +/// Only host, content-type, and content-length are included in the `SigV4` +/// signature. Signing all headers causes failures when the proxy or +/// transport modifies unsigned-by-convention headers (Connection, +/// Accept-Encoding, etc.) between signing and delivery. +/// +/// Header names are lowercased for comparison and stored in lowered form +/// in `all_headers`. This is intentional — AWS services accept +/// case-insensitive header names, and this function is only used on the +/// `SigV4` signing path. +fn parse_request_parts(header_str: &str) -> RequestParts<'_> { + // Headers stripped entirely — the SDK re-generates auth headers, and + // `Expect` is handled by the proxy before forwarding. + const STRIP_HEADERS: &[&str] = &[ + "authorization", + "x-amz-date", + "x-amz-security-token", + "x-amz-content-sha256", + "expect", + ]; + // Headers forwarded but NOT signed — the proxy or transport may modify + // them between signing and delivery, which would invalidate the signature. + const UNSIGNED_HEADERS: &[&str] = &[ + "connection", + "accept-encoding", + "transfer-encoding", + "user-agent", + "amz-sdk-invocation-id", + "amz-sdk-request", + ]; + + let lines: Vec<&str> = header_str.split("\r\n").collect(); + + let (method, path, request_line) = + lines + .first() + .map_or(("GET", "/", "GET / HTTP/1.1"), |first_line| { + let parts: Vec<&str> = first_line.splitn(3, ' ').collect(); + if parts.len() >= 2 { + (parts[0], parts[1], *first_line) + } else { + ("GET", "/", *first_line) + } + }); + + let mut headers_to_sign: Vec<(String, String)> = Vec::new(); + let mut all_headers: Vec<(String, String)> = Vec::new(); + for line in lines.iter().skip(1) { + if line.is_empty() { + break; + } + if let Some((k, v)) = line.split_once(':') { + let lower = k.trim().to_ascii_lowercase(); + if STRIP_HEADERS.iter().any(|s| lower.starts_with(s)) { + continue; + } + all_headers.push((lower.clone(), v.trim().to_string())); + if !UNSIGNED_HEADERS.iter().any(|s| lower.starts_with(s)) { + headers_to_sign.push((lower, v.trim().to_string())); + } + } + } + + RequestParts { + method, + path, + request_line, + headers_to_sign, + all_headers, + } +} + +fn build_signing_params<'a>( + identity: &'a Identity, + region: &'a str, + service: &'a str, +) -> Result> { + let mut settings = SigningSettings::default(); + settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; + + Ok(v4::SigningParams::builder() + .identity(identity) + .region(region) + .name(service) + .time(SystemTime::now()) + .settings(settings) + .build() + .map_err(|e| miette!("SigV4 signing params: {e}"))? + .into()) +} + +fn build_identity(access_key: &str, secret_key: &str, session_token: Option<&str>) -> Identity { + Credentials::new( + access_key, + secret_key, + session_token.map(ToString::to_string), + None, + "openshell", + ) + .into() +} + +fn rebuild_request( + parts: &RequestParts<'_>, + instructions: &aws_sigv4::http_request::SigningInstructions, + body: &[u8], +) -> Vec { + let mut output = Vec::with_capacity(256 + body.len()); + + output.extend_from_slice(parts.request_line.as_bytes()); + output.extend_from_slice(b"\r\n"); + + for (k, v) in &parts.all_headers { + output.extend_from_slice(format!("{k}: {v}\r\n").as_bytes()); + } + + for (name, value) in instructions.headers() { + output.extend_from_slice(format!("{name}: {value}\r\n").as_bytes()); + } + + output.extend_from_slice(b"\r\n"); + output.extend_from_slice(body); + + output +} + +/// Apply AWS Signature Version 4 signing to a raw HTTP request buffer. +/// +/// Strips existing AWS auth headers, computes a new signature using the +/// `aws-sigv4` crate, and returns the rewritten request bytes including body. +pub fn apply_sigv4_to_request( + raw: &[u8], + host: &str, + region: &str, + service: &str, + access_key: &str, + secret_key: &str, + session_token: Option<&str>, +) -> Result> { + let header_end = raw + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(raw.len(), |p| p + 4); + + let body = if header_end < raw.len() { + &raw[header_end..] + } else { + &[] + }; + + let header_str = std::str::from_utf8(&raw[..header_end]) + .map_err(|e| miette!("SigV4 signing: request headers are not valid UTF-8: {e}"))?; + let parts = parse_request_parts(header_str); + let uri = format!("https://{host}{}", parts.path); + let identity = build_identity(access_key, secret_key, session_token); + let signing_params = build_signing_params(&identity, region, service)?; + + let signable_request = SignableRequest::new( + parts.method, + &uri, + parts + .headers_to_sign + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())), + SignableBody::Bytes(body), + ) + .map_err(|e| miette!("SigV4 signable request: {e}"))?; + + let (instructions, _signature) = sign(signable_request, &signing_params) + .map_err(|e| miette!("SigV4 signing failed: {e}"))? + .into_parts(); + + Ok(rebuild_request(&parts, &instructions, body)) +} + +/// Apply AWS `SigV4` signing to HTTP headers only, using UNSIGNED-PAYLOAD. +/// +/// Returns signed headers ending with `\r\n\r\n`. The caller is responsible +/// for streaming the body separately. Use when the body is chunked or when +/// the service accepts unsigned payloads (e.g. S3 over HTTPS). +pub fn apply_sigv4_headers_only( + raw_headers: &[u8], + host: &str, + region: &str, + service: &str, + access_key: &str, + secret_key: &str, + session_token: Option<&str>, +) -> Result> { + apply_sigv4_headers_only_with_body( + raw_headers, + host, + region, + service, + access_key, + secret_key, + session_token, + SignableBody::UnsignedPayload, + ) +} + +/// Apply AWS `SigV4` signing to HTTP headers only with a caller-chosen +/// `SignableBody` mode (e.g. `UnsignedPayload` or +/// `StreamingUnsignedPayloadTrailer`). +/// +/// Returns signed headers ending with `\r\n\r\n`. +#[allow(clippy::too_many_arguments)] +pub fn apply_sigv4_headers_only_with_body( + raw_headers: &[u8], + host: &str, + region: &str, + service: &str, + access_key: &str, + secret_key: &str, + session_token: Option<&str>, + body: SignableBody<'_>, +) -> Result> { + let header_str = std::str::from_utf8(raw_headers) + .map_err(|e| miette!("SigV4 signing: request headers are not valid UTF-8: {e}"))?; + let parts = parse_request_parts(header_str); + let uri = format!("https://{host}{}", parts.path); + let identity = build_identity(access_key, secret_key, session_token); + let signing_params = build_signing_params(&identity, region, service)?; + + let signable_request = SignableRequest::new( + parts.method, + &uri, + parts + .headers_to_sign + .iter() + .map(|(k, v)| (k.as_str(), v.as_str())), + body, + ) + .map_err(|e| miette!("SigV4 signable request: {e}"))?; + + let (instructions, _signature) = sign(signable_request, &signing_params) + .map_err(|e| miette!("SigV4 signing failed: {e}"))? + .into_parts(); + + Ok(rebuild_request(&parts, &instructions, &[])) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extract_region_from_hostname() { + let region = extract_aws_region("bedrock-runtime.us-east-2.amazonaws.com").unwrap(); + assert_eq!(region, "us-east-2"); + } + + #[test] + fn extract_region_from_sts_hostname() { + let region = extract_aws_region("sts.us-east-1.amazonaws.com").unwrap(); + assert_eq!(region, "us-east-1"); + } + + #[test] + fn non_aws_hostname_returns_none() { + assert!(extract_aws_region("api.anthropic.com").is_none()); + } + + #[test] + fn global_endpoint_returns_none() { + assert!(extract_aws_region("s3.amazonaws.com").is_none()); + } + + #[test] + fn virtual_hosted_global_endpoint_returns_none() { + assert!(extract_aws_region("my-bucket.s3.amazonaws.com").is_none()); + } + + #[test] + fn extract_region_dualstack() { + let region = extract_aws_region("s3.dualstack.us-west-2.amazonaws.com").unwrap(); + assert_eq!(region, "us-west-2"); + } + + #[test] + fn extract_region_fips() { + let region = extract_aws_region("bedrock-runtime-fips.us-east-1.amazonaws.com").unwrap(); + assert_eq!(region, "us-east-1"); + } + + #[test] + fn extract_region_china() { + let region = extract_aws_region("s3.cn-north-1.amazonaws.com.cn").unwrap(); + assert_eq!(region, "cn-north-1"); + } + + #[test] + fn extract_region_fips_dualstack() { + let region = extract_aws_region("s3-fips.dualstack.us-west-2.amazonaws.com").unwrap(); + assert_eq!(region, "us-west-2"); + } + + #[test] + fn extract_region_govcloud() { + let region = extract_aws_region("s3.us-gov-west-1.amazonaws.com").unwrap(); + assert_eq!(region, "us-gov-west-1"); + } + + #[test] + fn extract_region_virtual_hosted_s3() { + let region = extract_aws_region("my-bucket.s3.us-east-2.amazonaws.com").unwrap(); + assert_eq!(region, "us-east-2"); + } + + #[test] + fn sign_produces_valid_format() { + let raw = b"POST /model/us.anthropic.claude-sonnet-4-6/invoke HTTP/1.1\r\nHost: bedrock-runtime.us-east-2.amazonaws.com\r\nContent-Type: application/json\r\n\r\n{}"; + let result = apply_sigv4_to_request( + raw, + "bedrock-runtime.us-east-2.amazonaws.com", + "us-east-2", + "bedrock", + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + None, + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!( + result_str.contains("authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") + ); + assert!(result_str.contains("x-amz-content-sha256: ")); + assert!(result_str.contains("x-amz-date: ")); + assert!(!result_str.contains("x-amz-security-token")); + } + + #[test] + fn sign_with_session_token() { + let raw = b"POST /model/test/invoke HTTP/1.1\r\nHost: bedrock-runtime.us-east-2.amazonaws.com\r\nContent-Type: application/json\r\n\r\n{}"; + let result = apply_sigv4_to_request( + raw, + "bedrock-runtime.us-east-2.amazonaws.com", + "us-east-2", + "bedrock", + "ASIAEXAMPLE", + "secret", + Some("FwoGZXIvYXdzEBYaDH+session+token"), + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!(result_str.contains("authorization: AWS4-HMAC-SHA256 Credential=ASIAEXAMPLE/")); + assert!(result_str.contains("x-amz-security-token: FwoGZXIvYXdzEBYaDH+session+token")); + } + + #[test] + fn non_signed_headers_preserved() { + let raw = b"POST /model/test/invoke HTTP/1.1\r\nHost: bedrock-runtime.us-east-2.amazonaws.com\r\nContent-Type: application/json\r\nAccept: application/json\r\nUser-Agent: my-agent/1.0\r\n\r\n{}"; + let result = apply_sigv4_to_request( + raw, + "bedrock-runtime.us-east-2.amazonaws.com", + "us-east-2", + "bedrock", + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + None, + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!(result_str.contains("accept: application/json\r\n")); + assert!(result_str.contains("user-agent: my-agent/1.0\r\n")); + assert!(result_str.contains("authorization: AWS4-HMAC-SHA256 Credential=")); + } + + #[test] + fn apply_sigv4_rewrites_request() { + let raw = b"POST /model/test/invoke HTTP/1.1\r\nHost: bedrock-runtime.us-east-2.amazonaws.com\r\nContent-Type: application/json\r\nAuthorization: AWS4-HMAC-SHA256 old-invalid-sig\r\nX-Amz-Date: old-date\r\n\r\n{}"; + let result = apply_sigv4_to_request( + raw, + "bedrock-runtime.us-east-2.amazonaws.com", + "us-east-2", + "bedrock", + "AKIATEST", + "secret", + None, + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!(result_str.contains("authorization: AWS4-HMAC-SHA256 Credential=AKIATEST/")); + assert!(!result_str.contains("old-invalid-sig")); + assert!(!result_str.contains("old-date")); + } + + #[test] + fn headers_only_produces_unsigned_payload() { + let raw = b"PUT /my-bucket/my-key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nContent-Type: application/octet-stream\r\nContent-Length: 1024\r\n\r\n"; + let result = apply_sigv4_headers_only( + raw, + "s3.us-east-1.amazonaws.com", + "us-east-1", + "s3", + "AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + None, + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!( + result_str.contains("authorization: AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/") + ); + assert!(result_str.contains("x-amz-content-sha256: UNSIGNED-PAYLOAD")); + assert!(result_str.contains("x-amz-date: ")); + assert!(result_str.ends_with("\r\n\r\n")); + } + + #[test] + fn headers_only_strips_old_auth() { + let raw = b"PUT /bucket/key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nAuthorization: AWS4-HMAC-SHA256 old-sig\r\nX-Amz-Date: old-date\r\nX-Amz-Content-Sha256: old-hash\r\nContent-Type: application/octet-stream\r\n\r\n"; + let result = apply_sigv4_headers_only( + raw, + "s3.us-east-1.amazonaws.com", + "us-east-1", + "s3", + "AKIATEST", + "secret", + None, + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!(result_str.contains("authorization: AWS4-HMAC-SHA256 Credential=AKIATEST/")); + assert!(!result_str.contains("old-sig")); + assert!(!result_str.contains("old-date")); + assert!(!result_str.contains("old-hash")); + assert!(result_str.contains("x-amz-content-sha256: UNSIGNED-PAYLOAD")); + } + + #[test] + fn headers_only_with_session_token() { + let raw = b"PUT /bucket/key HTTP/1.1\r\nHost: s3.us-east-1.amazonaws.com\r\nContent-Type: application/octet-stream\r\n\r\n"; + let result = apply_sigv4_headers_only( + raw, + "s3.us-east-1.amazonaws.com", + "us-east-1", + "s3", + "ASIAEXAMPLE", + "secret", + Some("FwoGZXIvYXdzEBYaDH+session+token"), + ) + .unwrap(); + let result_str = String::from_utf8_lossy(&result); + assert!(result_str.contains("x-amz-security-token: FwoGZXIvYXdzEBYaDH+session+token")); + assert!(result_str.contains("x-amz-content-sha256: UNSIGNED-PAYLOAD")); + } +} diff --git a/crates/openshell-supervisor-network/tests/sigv4_localstack.rs b/crates/openshell-supervisor-network/tests/sigv4_localstack.rs new file mode 100644 index 0000000000..2d76b7e2f7 --- /dev/null +++ b/crates/openshell-supervisor-network/tests/sigv4_localstack.rs @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Local-only integration test — not checked in. +// Requires LocalStack running on localhost:4566. +// Run with: cargo test -p openshell-supervisor-network --test sigv4_localstack -- --ignored --nocapture + +use openshell_supervisor_network::sigv4::{apply_sigv4_headers_only, apply_sigv4_to_request}; +use std::sync::atomic::{AtomicU32, Ordering}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +const LOCALSTACK: &str = "127.0.0.1:4566"; +const ACCESS_KEY: &str = "test"; +const SECRET_KEY: &str = "test"; +const REGION: &str = "us-east-1"; +const OBJECT_BODY: &str = "hello from sigv4 integration test"; + +static TEST_COUNTER: AtomicU32 = AtomicU32::new(0); + +fn unique_bucket() -> String { + let n = TEST_COUNTER.fetch_add(1, Ordering::SeqCst); + format!("sigv4-test-{n}-{}", std::process::id()) +} + +fn s3_host() -> String { + format!("s3.{REGION}.amazonaws.com") +} + +async fn send_raw(raw: &[u8]) -> (u16, String) { + let mut stream = TcpStream::connect(LOCALSTACK) + .await + .expect("connect to LocalStack"); + stream.write_all(raw).await.expect("write request"); + + let mut buf = Vec::with_capacity(16384); + let mut tmp = [0u8; 4096]; + loop { + match tokio::time::timeout(std::time::Duration::from_secs(5), stream.read(&mut tmp)).await { + Ok(Ok(0)) | Err(_) => break, + Ok(Ok(n)) => buf.extend_from_slice(&tmp[..n]), + Ok(Err(e)) => panic!("read error: {e}"), + } + } + + let response = String::from_utf8_lossy(&buf).to_string(); + let status = response + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + + (status, response) +} + +async fn signed_request(raw: &str, host: &str, service: &str) -> (u16, String) { + let signed = apply_sigv4_to_request( + raw.as_bytes(), + host, + REGION, + service, + ACCESS_KEY, + SECRET_KEY, + None, + ) + .expect("signing failed"); + send_raw(&signed).await +} + +async fn create_bucket(bucket: &str) { + let raw = format!( + "PUT /{bucket} HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" + ); + let (status, body) = signed_request(&raw, &s3_host(), "s3").await; + assert!( + status == 200 || status == 409, + "CreateBucket: expected 200 or 409, got {status}\n{body}" + ); +} + +async fn put_object(bucket: &str, key: &str, body: &str) { + let raw = format!( + "PUT /{bucket}/{key} HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\nContent-Type: text/plain\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + let (status, response) = signed_request(&raw, &s3_host(), "s3").await; + assert!( + status == 200 || status == 204, + "PutObject: expected 200/204, got {status}\n{response}" + ); +} + +async fn delete_object(bucket: &str, key: &str) { + let raw = format!( + "DELETE /{bucket}/{key} HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" + ); + let (status, response) = signed_request(&raw, &s3_host(), "s3").await; + assert!( + status == 200 || status == 204, + "DeleteObject: expected 200/204, got {status}\n{response}" + ); +} + +async fn delete_bucket(bucket: &str) { + let raw = format!( + "DELETE /{bucket} HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n" + ); + let (status, response) = signed_request(&raw, &s3_host(), "s3").await; + assert!( + status == 200 || status == 204, + "DeleteBucket: expected 200/204, got {status}\n{response}" + ); +} + +#[tokio::test] +#[ignore = "requires LocalStack on localhost:4566"] +async fn sigv4_s3_create_bucket() { + let bucket = unique_bucket(); + create_bucket(&bucket).await; + delete_bucket(&bucket).await; + println!("PASS: S3 CreateBucket with SigV4 signed body"); +} + +#[tokio::test] +#[ignore = "requires LocalStack on localhost:4566"] +async fn sigv4_s3_put_and_delete_object() { + let bucket = unique_bucket(); + create_bucket(&bucket).await; + put_object(&bucket, "test.txt", OBJECT_BODY).await; + delete_object(&bucket, "test.txt").await; + delete_bucket(&bucket).await; + println!("PASS: S3 PutObject + DeleteObject with SigV4 signed body"); +} + +#[tokio::test] +#[ignore = "requires LocalStack on localhost:4566"] +async fn sigv4_s3_get_object_unsigned_payload() { + let bucket = unique_bucket(); + create_bucket(&bucket).await; + put_object(&bucket, "get-test.txt", OBJECT_BODY).await; + + let raw = format!( + "GET /{bucket}/get-test.txt HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\n\r\n" + ); + let signed = apply_sigv4_headers_only( + raw.as_bytes(), + &s3_host(), + REGION, + "s3", + ACCESS_KEY, + SECRET_KEY, + None, + ) + .expect("signing failed"); + + let (status, response) = send_raw(&signed).await; + assert_eq!( + status, 200, + "GetObject: expected 200, got {status}\n{response}" + ); + assert!( + response.contains(OBJECT_BODY), + "GetObject: body should contain '{OBJECT_BODY}'\n{response}" + ); + + delete_object(&bucket, "get-test.txt").await; + delete_bucket(&bucket).await; + println!("PASS: S3 GetObject with UNSIGNED-PAYLOAD (apply_sigv4_headers_only)"); +} + +#[tokio::test] +#[ignore = "requires LocalStack on localhost:4566"] +async fn sigv4_sts_get_caller_identity() { + let body = "Action=GetCallerIdentity&Version=2011-06-15"; + let raw = format!( + "POST / HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\nContent-Type: application/x-www-form-urlencoded\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ); + + let signed = apply_sigv4_to_request( + raw.as_bytes(), + &format!("sts.{REGION}.amazonaws.com"), + REGION, + "sts", + ACCESS_KEY, + SECRET_KEY, + None, + ) + .expect("signing failed"); + + let (status, response) = send_raw(&signed).await; + assert_eq!( + status, 200, + "GetCallerIdentity: expected 200, got {status}\n{response}" + ); + assert!( + response.contains("GetCallerIdentityResult"), + "GetCallerIdentity: response should contain result\n{response}" + ); + println!("PASS: STS GetCallerIdentity with SigV4 signed body"); +} + +#[tokio::test] +#[ignore = "requires LocalStack on localhost:4566"] +async fn sigv4_s3_put_with_session_token() { + let bucket = unique_bucket(); + create_bucket(&bucket).await; + + let raw = format!( + "PUT /{bucket}/session-test.txt HTTP/1.1\r\nHost: {LOCALSTACK}\r\nConnection: close\r\nContent-Type: text/plain\r\nContent-Length: 12\r\n\r\nsession-data" + ); + let signed = apply_sigv4_to_request( + raw.as_bytes(), + &s3_host(), + REGION, + "s3", + ACCESS_KEY, + SECRET_KEY, + Some("FakeSessionToken"), + ) + .expect("signing with session token failed"); + + let signed_str = String::from_utf8_lossy(&signed); + assert!( + signed_str.contains("x-amz-security-token: FakeSessionToken"), + "signed request should include session token header" + ); + + let (status, response) = send_raw(&signed).await; + assert!( + status == 200 || status == 204, + "PutObject with session token: expected 200/204, got {status}\n{response}" + ); + + delete_object(&bucket, "session-test.txt").await; + delete_bucket(&bucket).await; + println!("PASS: S3 PutObject with session token"); +} diff --git a/docs/providers/aws-sigv4.mdx b/docs/providers/aws-sigv4.mdx new file mode 100644 index 0000000000..fc5e02040b --- /dev/null +++ b/docs/providers/aws-sigv4.mdx @@ -0,0 +1,151 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: "AWS SigV4 Credential Signing" +sidebar-title: "AWS SigV4" +description: "Configure proxy-side AWS SigV4 request signing so sandbox agents can reach AWS services through CONNECT tunnels without holding real credentials." +keywords: "Generative AI, Cybersecurity, AI Agents, AWS, SigV4, Bedrock, S3, Credential Signing, Sandbox" +--- + +AWS SigV4 credential signing lets sandbox agents call AWS services (Bedrock, S3, STS, and others) through the proxy's CONNECT tunnel. The proxy intercepts outbound requests, strips the sandbox client's placeholder `Authorization` header, and re-signs the request with real AWS credentials from the provider. The sandbox never sees the real credentials. + +## Prerequisites + +- A provider with `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` credentials configured. Optionally include `AWS_SESSION_TOKEN` for STS temporary credentials. +- A sandbox policy with `credential_signing` enabled on the target endpoint. + +## Provider Setup + +Create a provider with AWS credentials: + +```shell +openshell provider create \ + --name aws-prod \ + --credential AWS_ACCESS_KEY_ID=AKIA... \ + --credential AWS_SECRET_ACCESS_KEY=wJalr... +``` + +For STS temporary credentials, include the session token: + +```shell +openshell provider create \ + --name aws-sts \ + --credential AWS_ACCESS_KEY_ID=ASIA... \ + --credential AWS_SECRET_ACCESS_KEY=secret... \ + --credential AWS_SESSION_TOKEN=FwoGZX... +``` + +## Policy Configuration + +Enable SigV4 signing on a per-endpoint basis using three policy fields: + +| Field | Type | Required | Description | +|---|---|---|---| +| `credential_signing` | string | Yes | Signing mode: `sigv4`, `sigv4:body`, or `sigv4:no_body`. | +| `signing_service` | string | Yes | AWS service name for the SigV4 signature (e.g. `bedrock`, `s3`, `sts`). | +| `signing_region` | string | No | AWS region override. When omitted, extracted from the endpoint hostname. Required for non-standard endpoints. | + +### Bedrock Example + +```yaml +network_policies: + aws_bedrock: + endpoints: + - host: bedrock-runtime.us-east-1.amazonaws.com + port: 443 + protocol: rest + credential_signing: sigv4 + signing_service: bedrock + rules: + - allow: + method: POST + path: /model/*/invoke +``` + + +The Bedrock example uses `rules` for fine-grained access control. When `rules` are present, omit the `access` field — they are mutually exclusive. + + +### S3 Example + +```yaml +network_policies: + aws_s3: + endpoints: + - host: "*.s3.us-east-1.amazonaws.com" + port: 443 + protocol: rest + access: full + credential_signing: sigv4 + signing_service: s3 +``` + +### STS Example + +```yaml +network_policies: + aws_sts: + endpoints: + - host: sts.us-east-1.amazonaws.com + port: 443 + protocol: rest + access: full + credential_signing: sigv4 + signing_service: sts +``` + +## Signing Modes + +The `credential_signing` field accepts three values: + +| Value | Behavior | Use When | +|---|---|---| +| `sigv4` | Auto-detect payload mode from the client SDK's `x-amz-content-sha256` header. | Default. Works for most AWS services. | +| `sigv4:body` | Always buffer the request body and include its SHA-256 hash in the signature. Maximum body size: 10 MiB. | Services that require body signing (Bedrock). | +| `sigv4:no_body` | Sign headers only with `UNSIGNED-PAYLOAD`. Stream the body through without buffering. | Large uploads (S3 PutObject), chunked transfers, or any case where body buffering is impractical. | + +In `sigv4` auto-detect mode, the proxy inspects the `x-amz-content-sha256` header sent by the client SDK: + +- Hex hash → buffer body and sign it (same as `sigv4:body`). +- `UNSIGNED-PAYLOAD` → sign headers only (same as `sigv4:no_body`). +- `STREAMING-UNSIGNED-PAYLOAD-TRAILER` → sign headers only, stream body through. +- Absent → sign body if `Content-Length` is present, otherwise use unsigned payload. + + +Chunk-signed streaming modes like `STREAMING-AWS4-HMAC-SHA256-PAYLOAD` are not supported. The proxy cannot reproduce per-chunk signatures. If your client SDK sends chunk-signed requests, use `sigv4:no_body` instead. + + +## Region Detection + +The proxy extracts the AWS region from the endpoint hostname automatically. It supports standard, dualstack, FIPS, virtual-hosted, GovCloud, and China partition hostnames. + +For endpoints where the region cannot be inferred from the hostname, set `signing_region` explicitly: + +```yaml +endpoints: + - host: custom-vpc-endpoint.example.com + port: 443 + protocol: rest + access: full + credential_signing: sigv4 + signing_service: s3 + signing_region: us-west-2 +``` + +## Restrictions + +- `credential_signing` and `request_body_credential_rewrite` are mutually exclusive on the same endpoint. The policy validator rejects policies that set both. +- The `sigv4:body` mode buffers at most 10 MiB. Requests with larger bodies are rejected. Use `sigv4:no_body` or `sigv4` (auto-detect) for large payloads. +- The proxy requires `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in the provider. If either is missing, the request fails with an error. + +## Use from a Sandbox + +Inside a sandbox, configure the AWS SDK with placeholder credentials. The proxy replaces them with real credentials during re-signing: + +```shell +export AWS_ACCESS_KEY_ID=placeholder +export AWS_SECRET_ACCESS_KEY=placeholder +export AWS_DEFAULT_REGION=us-east-1 +``` + +Then use any AWS SDK or CLI normally. The proxy transparently re-signs requests before forwarding to AWS. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 59f72c9f75..5dc8d9a5ed 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -164,7 +164,10 @@ Each endpoint defines a reachable destination and optional inspection rules. | `allowed_ips` | list of string | No | CIDR or IP allowlist for SSRF override. Exact user-declared hostname endpoints may resolve to RFC 1918 private addresses without this field, but wildcard, hostless, and policy-advisor-proposed endpoints still require `allowed_ips` for private resolved IPs. Entries overlapping loopback (`127.0.0.0/8`), link-local (`169.254.0.0/16`), or unspecified (`0.0.0.0`) are rejected at load time. | | `allow_encoded_slash` | bool | No | When `true`, L7 request parsing preserves `%2F` inside path segments instead of rejecting it. Use this for registries and APIs such as npm scoped packages (`/@scope%2Fname`). Defaults to `false`. | | `websocket_credential_rewrite` | bool | No | When `true` on a `protocol: rest` or `protocol: websocket` endpoint, OpenShell rewrites credential placeholders in client-to-server WebSocket text messages after an allowed HTTP `101` upgrade. Binary frames are relayed but not rewritten. Defaults to `false`. | -| `request_body_credential_rewrite` | bool | No | When `true` on a `protocol: rest` endpoint, OpenShell rewrites credential placeholders in UTF-8 `application/json`, `application/x-www-form-urlencoded`, and `text/*` request bodies before forwarding upstream. The proxy buffers at most 256 KiB and updates `Content-Length` after rewriting. Defaults to `false`. | +| `request_body_credential_rewrite` | bool | No | When `true` on a `protocol: rest` endpoint, OpenShell rewrites credential placeholders in UTF-8 `application/json`, `application/x-www-form-urlencoded`, and `text/*` request bodies before forwarding upstream. The proxy buffers at most 256 KiB and updates `Content-Length` after rewriting. Defaults to `false`. Mutually exclusive with `credential_signing`. | +| `credential_signing` | string | No | Proxy-side credential signing mode. When set, the proxy strips the sandbox client's `Authorization` header and re-signs with real provider credentials. Values: `sigv4` (auto-detect payload mode from client headers), `sigv4:body` (buffer and hash body, max 10 MiB), `sigv4:no_body` (unsigned payload, stream body). Mutually exclusive with `request_body_credential_rewrite`. See [AWS SigV4](/providers/aws-sigv4). | +| `signing_service` | string | No | AWS service name for SigV4 signing (e.g. `bedrock`, `s3`, `sts`). Required when `credential_signing` is set. | +| `signing_region` | string | No | AWS region override for SigV4 signing (e.g. `us-east-1`). When omitted, the region is extracted from the endpoint hostname. Required for non-standard AWS endpoints where the region cannot be inferred. | | `persisted_queries` | string | No | GraphQL hash-only behavior for `protocol: graphql` and GraphQL-over-WebSocket operation policy. Default is `deny`; use `allow_registered` only with `graphql_persisted_queries`. | | `graphql_persisted_queries` | map | No | Trusted GraphQL persisted-query registry keyed by hash or saved-query ID. Values contain `operation_type`, optional `operation_name`, and optional root `fields`. | | `graphql_max_body_bytes` | integer | No | Maximum GraphQL-over-HTTP request body bytes buffered for inspection. Defaults to `65536`. | diff --git a/proto/sandbox.proto b/proto/sandbox.proto index ef0b0540f7..faf7569acf 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -128,6 +128,16 @@ message NetworkEndpoint { // Advisor-proposed endpoints must not satisfy exact-host SSRF trust unless // they are converted through an explicit user-authored policy path. bool advisor_proposed = 18; + // Proxy-side credential signing mode: "sigv4" for AWS SigV4 re-signing. + // When set, the proxy strips the client's Authorization header and computes + // a fresh SigV4 signature using real credentials from the provider. + string credential_signing = 19; + // AWS signing service name override. Required when credential_signing is + // "sigv4" — e.g. "bedrock" for bedrock-runtime endpoints. + string signing_service = 20; + // AWS region override for SigV4 signing. When set, takes precedence over + // hostname-based region extraction. Required for non-standard endpoints. + string signing_region = 21; } // Trusted GraphQL operation classification. From ba21bb32a2c04f091b64a0c593d66c6ff34cf382 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 26 Jun 2026 11:44:52 -0700 Subject: [PATCH 15/20] feat(kubernetes): support agent-sandbox v1beta1 (#2009) * feat(kubernetes): support agent-sandbox v1beta1 Signed-off-by: Taylor Mutch * ci(kubernetes): test agent-sandbox api versions Signed-off-by: Taylor Mutch * fix(kubernetes): retry agent-sandbox raw 404s Signed-off-by: Taylor Mutch * fix(kubernetes): harden agent-sandbox api setup Signed-off-by: Taylor Mutch * fix(kubernetes): cache agent-sandbox api version Signed-off-by: Taylor Mutch * docs(kubernetes): document agent-sandbox upgrade behavior Signed-off-by: Taylor Mutch * refactor(kubernetes): collapse agent-sandbox api selection Signed-off-by: Taylor Mutch --------- Signed-off-by: Taylor Mutch --- .github/workflows/branch-e2e.yml | 10 + .github/workflows/e2e-kubernetes-test.yml | 6 + architecture/gateway.md | 11 +- crates/openshell-driver-kubernetes/README.md | 10 +- .../openshell-driver-kubernetes/src/driver.rs | 173 +++++++++++++++--- crates/openshell-server/src/auth/k8s_sa.rs | 170 +++++++++++++++-- docs/kubernetes/setup.mdx | 4 + docs/reference/sandbox-compute-drivers.mdx | 4 +- e2e/with-kube-gateway.sh | 31 +++- tasks/scripts/helm-k3s-local.sh | 7 +- tasks/test.toml | 12 ++ 11 files changed, 382 insertions(+), 56 deletions(-) diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 23d7d1cf6f..ebe7834068 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -115,12 +115,22 @@ jobs: kubernetes-e2e: needs: [pr_metadata, build-gateway, build-supervisor] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + strategy: + fail-fast: false + matrix: + include: + - agent_sandbox_api: v1beta1 + agent_sandbox_version: v0.5.0 + - agent_sandbox_api: v1alpha1 + agent_sandbox_version: v0.4.6 permissions: contents: read packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) + agent-sandbox-version: ${{ matrix.agent_sandbox_version }} kubernetes-ha-e2e: needs: [pr_metadata, build-gateway, build-supervisor] diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index 5ff3759222..f8c042b2a6 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -32,6 +32,11 @@ on: required: false type: string default: "" + agent-sandbox-version: + description: "Agent Sandbox release to install before OpenShell" + required: false + type: string + default: "v0.5.0" mise-version: description: "mise version to install on the bare Kubernetes e2e runner" required: false @@ -114,6 +119,7 @@ jobs: - name: Run Kubernetes E2E (Rust smoke) env: + AGENT_SANDBOX_VERSION: ${{ inputs.agent-sandbox-version }} OPENSHELL_E2E_KUBE_CONTEXT: kind-${{ env.KIND_CLUSTER_NAME }} OPENSHELL_E2E_KUBE_EXTRA_VALUES: ${{ inputs.extra-helm-values }} OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET: ${{ inputs.external-postgres-secret }} diff --git a/architecture/gateway.md b/architecture/gateway.md index 979422d7e8..2e02562491 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -66,10 +66,13 @@ token through `IssueSandboxToken`. The gateway validates that projected token with Kubernetes `TokenReview`, requires the configured sandbox service account, checks the returned pod binding against the live pod UID, and verifies the pod's controlling `Sandbox` ownerReference against the live Sandbox CR UID and -sandbox-id label before minting the gateway JWT. Supervisors renew gateway JWTs -in memory before expiry only while the sandbox record still exists. Older tokens -are not server-revoked; shared deployments bound replay exposure with short -`gateway_jwt.ttl_secs` lifetimes. The config default is +sandbox-id label before minting the gateway JWT. The bootstrap path accepts +both `agents.x-k8s.io/v1beta1` ownerReferences from newer Agent Sandbox +controllers and `agents.x-k8s.io/v1alpha1` ownerReferences from existing +deployments. Supervisors renew gateway JWTs in memory before expiry only while +the sandbox record still exists. Older tokens are not server-revoked; shared +deployments bound replay exposure with short `gateway_jwt.ttl_secs` lifetimes. +The config default is `gateway_jwt.ttl_secs = 0` for local single-player Docker, Podman, and VM gateways; those tokens carry `exp = 0` and do not expire. Kubernetes and other shared deployments should set a positive TTL. diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 3cdb9fa570..831e4edf29 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -15,9 +15,13 @@ credential injection, policy polling, logs, and the gateway relay. ## Sandbox Resource -The driver works with the `agents.x-k8s.io/v1alpha1` `Sandbox` custom resource. -Driver events map Kubernetes object state and platform events into the shared -compute-driver protobuf surface used by the gateway. +The driver works with the `agents.x-k8s.io` `Sandbox` custom resource. It +detects the served Sandbox API at runtime, caches the selected API version for +the gateway process, and uses `v1beta1` when available before falling back to +`v1alpha1`. Restart the gateway after an in-place Agent Sandbox upgrade so the +driver can detect served API versions again. Driver events map Kubernetes object +state and platform events into the shared compute-driver protobuf surface used +by the gateway. Kubernetes API calls use explicit timeouts so gRPC handlers do not block indefinitely when the API server is slow or unavailable. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index a32034f3a3..9095683023 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -34,8 +34,9 @@ use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use serde::Deserialize; use std::collections::BTreeMap; use std::pin::Pin; +use std::sync::Arc; use std::time::Duration; -use tokio::sync::mpsc; +use tokio::sync::{OnceCell, mpsc}; use tokio_stream::wrappers::ReceiverStream; use tracing::{debug, info, warn}; @@ -80,12 +81,19 @@ impl From for openshell_core::ComputeDriverError { const KUBE_API_TIMEOUT: Duration = Duration::from_secs(30); const SANDBOX_GROUP: &str = "agents.x-k8s.io"; -const SANDBOX_VERSION: &str = "v1alpha1"; +const SANDBOX_VERSION_V1BETA1: &str = "v1beta1"; +const SANDBOX_VERSION_V1ALPHA1: &str = "v1alpha1"; +const SANDBOX_VERSIONS: &[&str] = &[SANDBOX_VERSION_V1BETA1, SANDBOX_VERSION_V1ALPHA1]; pub const SANDBOX_KIND: &str = "Sandbox"; const GPU_RESOURCE_NAME: &str = "nvidia.com/gpu"; const SPIFFE_WORKLOAD_API_VOLUME_NAME: &str = "spiffe-workload-api"; +struct AgentSandboxApi { + api: Api, + resource: ApiResource, +} + // This POC treats the selected Struct as a driver-local typed schema. Once the // Kubernetes shape stabilizes, these serde structs may move to driver-local // protobuf definitions, but the typed decode should stay inside this driver. @@ -190,6 +198,7 @@ const WORKSPACE_SENTINEL: &str = ".workspace-initialized"; pub struct KubernetesComputeDriver { client: Client, watch_client: Client, + sandbox_api_version: Arc>, config: KubernetesComputeConfig, } @@ -232,6 +241,7 @@ impl KubernetesComputeDriver { Ok(Self { client, watch_client, + sandbox_api_version: Arc::new(OnceCell::new()), config, }) } @@ -256,16 +266,68 @@ impl KubernetesComputeDriver { &self.config.ssh_socket_path } - fn watch_api(&self) -> Api { - let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND); + fn agent_sandbox_api(&self, client: Client, sandbox_api_version: &str) -> AgentSandboxApi { + let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); let resource = ApiResource::from_gvk(&gvk); - Api::namespaced_with(self.watch_client.clone(), &self.config.namespace, &resource) + let api = Api::namespaced_with(client, &self.config.namespace, &resource); + AgentSandboxApi { api, resource } } - fn api(&self) -> Api { - let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND); - let resource = ApiResource::from_gvk(&gvk); - Api::namespaced_with(self.client.clone(), &self.config.namespace, &resource) + async fn supported_agent_sandbox_api(&self, client: Client) -> Result { + let sandbox_api_version = self.supported_sandbox_api_version(client.clone()).await?; + Ok(self.agent_sandbox_api(client, sandbox_api_version)) + } + + async fn supported_sandbox_api_version(&self, client: Client) -> Result<&'static str, String> { + self.sandbox_api_version + .get_or_try_init( + || async move { self.detect_supported_sandbox_api_version(client).await }, + ) + .await + .copied() + } + + async fn detect_supported_sandbox_api_version( + &self, + client: Client, + ) -> Result<&'static str, String> { + for sandbox_api_version in SANDBOX_VERSIONS { + let agent_sandbox_api = self.agent_sandbox_api(client.clone(), sandbox_api_version); + match tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.list(&ListParams::default().limit(1)), + ) + .await + { + Ok(Ok(_)) => { + debug!( + namespace = %self.config.namespace, + sandbox_api_version = %sandbox_api_version, + "Selected Agent Sandbox API version" + ); + return Ok(sandbox_api_version); + } + Ok(Err(err)) if should_try_next_sandbox_api_version(&err) => { + debug!( + namespace = %self.config.namespace, + sandbox_api_version = %sandbox_api_version, + error = %err, + "Sandbox API version is not available; trying next supported version" + ); + } + Ok(Err(err)) => return Err(err.to_string()), + Err(_elapsed) => { + return Err(format!( + "timed out after {}s waiting for Kubernetes API", + KUBE_API_TIMEOUT.as_secs() + )); + } + } + } + Err(format!( + "no supported Agent Sandbox API version is available; tried {}", + SANDBOX_VERSIONS.join(", ") + )) } async fn has_gpu_capacity(&self) -> Result { @@ -306,8 +368,10 @@ impl KubernetesComputeDriver { "Fetching sandbox from Kubernetes" ); - let api = self.api(); - match tokio::time::timeout(KUBE_API_TIMEOUT, api.get(name)).await { + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.client.clone()) + .await?; + match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.get(name)).await { Ok(Ok(obj)) => sandbox_from_object(&self.config.namespace, obj).map(Some), Ok(Err(KubeError::Api(err))) if err.code == 404 => { debug!(sandbox_name = %name, "Sandbox not found in Kubernetes"); @@ -341,8 +405,15 @@ impl KubernetesComputeDriver { "Listing sandboxes from Kubernetes" ); - let api = self.api(); - match tokio::time::timeout(KUBE_API_TIMEOUT, api.list(&ListParams::default())).await { + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.client.clone()) + .await?; + match tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.list(&ListParams::default()), + ) + .await + { Ok(Ok(list)) => { let mut sandboxes = list .items @@ -396,9 +467,11 @@ impl KubernetesComputeDriver { "Creating sandbox in Kubernetes" ); - let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND); - let resource = ApiResource::from_gvk(&gvk); - let mut obj = DynamicObject::new(name, &resource); + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.client.clone()) + .await + .map_err(KubernetesDriverError::Message)?; + let mut obj = DynamicObject::new(name, &agent_sandbox_api.resource); obj.metadata = ObjectMeta { name: Some(name.to_string()), namespace: Some(self.config.namespace.clone()), @@ -430,9 +503,11 @@ impl KubernetesComputeDriver { .provider_spiffe_workload_api_socket_path, }; obj.data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms); - let api = self.api(); - - match tokio::time::timeout(KUBE_API_TIMEOUT, api.create(&PostParams::default(), &obj)).await + match tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.create(&PostParams::default(), &obj), + ) + .await { Ok(Ok(_result)) => { info!( @@ -473,9 +548,14 @@ impl KubernetesComputeDriver { "Deleting sandbox from Kubernetes" ); - let api = self.api(); - match tokio::time::timeout(KUBE_API_TIMEOUT, api.delete(name, &DeleteParams::default())) - .await + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.client.clone()) + .await?; + match tokio::time::timeout( + KUBE_API_TIMEOUT, + agent_sandbox_api.api.delete(name, &DeleteParams::default()), + ) + .await { Ok(Ok(_response)) => { info!(sandbox_name = %name, "Sandbox deleted from Kubernetes"); @@ -508,8 +588,10 @@ impl KubernetesComputeDriver { } pub async fn sandbox_exists(&self, name: &str) -> Result { - let api = self.api(); - match tokio::time::timeout(KUBE_API_TIMEOUT, api.get(name)).await { + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.client.clone()) + .await?; + match tokio::time::timeout(KUBE_API_TIMEOUT, agent_sandbox_api.api.get(name)).await { Ok(Ok(_)) => Ok(true), Ok(Err(KubeError::Api(err))) if err.code == 404 => Ok(false), Ok(Err(err)) => Err(err.to_string()), @@ -524,9 +606,12 @@ impl KubernetesComputeDriver { #[allow(clippy::unused_async)] pub async fn watch_sandboxes(&self) -> Result { let namespace = self.config.namespace.clone(); - let sandbox_api = self.watch_api(); + let agent_sandbox_api = self + .supported_agent_sandbox_api(self.watch_client.clone()) + .await?; let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); - let mut sandbox_stream = watcher::watcher(sandbox_api, watcher::Config::default()).boxed(); + let mut sandbox_stream = + watcher::watcher(agent_sandbox_api.api, watcher::Config::default()).boxed(); let mut event_stream = watcher::watcher(event_api, watcher::Config::default()).boxed(); let (tx, rx) = mpsc::channel(256); @@ -650,6 +735,14 @@ impl KubernetesComputeDriver { } } +fn should_try_next_sandbox_api_version(err: &KubeError) -> bool { + // Kubernetes returns a structured 404 for some missing API resources and a + // raw "404 page not found" body for others. Both mean the probed + // group/version is unavailable and the next supported Sandbox API version + // should be tried. + matches!(err, KubeError::Api(api) if api.code == 404) +} + fn validate_gpu_request( gpu_requirements: Option<&GpuResourceRequirements>, ) -> Result<(), tonic::Status> { @@ -2013,6 +2106,34 @@ mod tests { } } + fn kube_api_error(code: u16, message: &str) -> KubeError { + KubeError::Api(kube::core::ErrorResponse { + status: if code == 404 { + "404 Not Found".to_string() + } else { + "Failure".to_string() + }, + message: message.to_string(), + reason: "Failed to parse error data".to_string(), + code, + }) + } + + #[test] + fn sandbox_api_version_probe_retries_on_structured_and_raw_404() { + let structured = kube_api_error(404, "could not find the requested resource"); + assert!(should_try_next_sandbox_api_version(&structured)); + + let raw = kube_api_error(404, "404 page not found\n"); + assert!(should_try_next_sandbox_api_version(&raw)); + } + + #[test] + fn sandbox_api_version_probe_keeps_non_404_errors() { + let err = kube_api_error(403, "sandboxes.agents.x-k8s.io is forbidden"); + assert!(!should_try_next_sandbox_api_version(&err)); + } + #[test] fn driver_config_rejects_invalid_shape() { let template = SandboxTemplate { diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 59f6651bb1..eed0e5f083 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -23,6 +23,7 @@ use k8s_openapi::api::{ core::v1::Pod, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; +use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; use std::sync::Arc; @@ -40,8 +41,10 @@ pub const ISSUE_SANDBOX_TOKEN_PATH: &str = "/openshell.v1.OpenShell/IssueSandbox /// include `patch pods` (see plan §11.8). pub const SANDBOX_ID_ANNOTATION: &str = "openshell.io/sandbox-id"; const SANDBOX_API_GROUP: &str = "agents.x-k8s.io"; -const SANDBOX_API_VERSION: &str = "v1alpha1"; -const SANDBOX_API_VERSION_FULL: &str = "agents.x-k8s.io/v1alpha1"; +const SANDBOX_API_VERSION_V1BETA1: &str = "v1beta1"; +const SANDBOX_API_VERSION_V1ALPHA1: &str = "v1alpha1"; +const SANDBOX_API_VERSION_FULL_V1BETA1: &str = "agents.x-k8s.io/v1beta1"; +const SANDBOX_API_VERSION_FULL_V1ALPHA1: &str = "agents.x-k8s.io/v1alpha1"; const SANDBOX_KIND: &str = "Sandbox"; const SANDBOX_ID_LABEL: &str = "openshell.ai/sandbox-id"; const POD_NAME_EXTRA: &str = "authentication.kubernetes.io/pod-name"; @@ -140,6 +143,7 @@ struct TokenReviewIdentity { #[derive(Debug, Clone, PartialEq, Eq)] struct SandboxOwnerReference { + api_version: String, name: String, uid: String, } @@ -149,7 +153,8 @@ struct SandboxOwnerReference { pub struct LiveK8sResolver { token_reviews_api: Api, pods_api: Api, - sandboxes_api: Api, + sandboxes_api_v1beta1: Api, + sandboxes_api_v1alpha1: Api, expected_audience: String, sandbox_namespace: String, expected_service_account: String, @@ -164,20 +169,51 @@ impl LiveK8sResolver { ) -> Self { let token_reviews_api: Api = Api::all(client.clone()); let pods_api: Api = Api::namespaced(client.clone(), namespace); - let sandbox_gvk = - GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION, SANDBOX_KIND); - let sandbox_resource = ApiResource::from_gvk(&sandbox_gvk); - let sandboxes_api: Api = - Api::namespaced_with(client, namespace, &sandbox_resource); + let sandbox_gvk_v1beta1 = + GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION_V1BETA1, SANDBOX_KIND); + let sandbox_resource_v1beta1 = ApiResource::from_gvk(&sandbox_gvk_v1beta1); + let sandbox_gvk_v1alpha1 = GroupVersionKind::gvk( + SANDBOX_API_GROUP, + SANDBOX_API_VERSION_V1ALPHA1, + SANDBOX_KIND, + ); + let sandbox_resource_v1alpha1 = ApiResource::from_gvk(&sandbox_gvk_v1alpha1); + let sandboxes_api_v1beta1: Api = + Api::namespaced_with(client.clone(), namespace, &sandbox_resource_v1beta1); + let sandboxes_api_v1alpha1: Api = + Api::namespaced_with(client, namespace, &sandbox_resource_v1alpha1); Self { token_reviews_api, pods_api, - sandboxes_api, + sandboxes_api_v1beta1, + sandboxes_api_v1alpha1, expected_audience, sandbox_namespace: namespace.to_string(), expected_service_account, } } + + async fn get_sandbox_cr_for_owner( + &self, + owner: &SandboxOwnerReference, + ) -> Result, KubeError> { + let apis = if owner.api_version == SANDBOX_API_VERSION_FULL_V1ALPHA1 { + [&self.sandboxes_api_v1alpha1, &self.sandboxes_api_v1beta1] + } else { + [&self.sandboxes_api_v1beta1, &self.sandboxes_api_v1alpha1] + }; + + for api in apis { + match api.get_opt(&owner.name).await { + Ok(Some(sandbox_cr)) => return Ok(Some(sandbox_cr)), + Ok(None) => {} + Err(err) if should_try_next_sandbox_api_version(&err) => {} + Err(err) => return Err(err), + } + } + + Ok(None) + } } #[async_trait] @@ -258,10 +294,11 @@ impl K8sIdentityResolver for LiveK8sResolver { let sandbox_id = pod_sandbox_id(&pod)?; let owner = sandbox_owner_reference(&pod)?; - let sandbox_cr = self.sandboxes_api.get_opt(&owner.name).await.map_err(|e| { + let sandbox_cr = self.get_sandbox_cr_for_owner(&owner).await.map_err(|e| { warn!( pod = %identity.pod_name, sandbox_owner = %owner.name, + sandbox_owner_api_version = %owner.api_version, error = %e, "failed to fetch owning Sandbox CR for pod identity validation" ); @@ -271,6 +308,7 @@ impl K8sIdentityResolver for LiveK8sResolver { warn!( pod = %identity.pod_name, sandbox_owner = %owner.name, + sandbox_owner_api_version = %owner.api_version, "pod ownerReference points to a Sandbox CR that does not exist" ); return Err(Status::permission_denied("sandbox owner not found")); @@ -370,10 +408,25 @@ fn pod_sandbox_id(pod: &Pod) -> Result { #[allow(clippy::result_large_err)] fn sandbox_owner_reference(pod: &Pod) -> Result { let owner_refs = pod.metadata.owner_references.as_deref().unwrap_or_default(); - let mut sandbox_refs = owner_refs.iter().filter(|owner| { - owner.api_version == SANDBOX_API_VERSION_FULL && owner.kind == SANDBOX_KIND - }); + let mut sandbox_refs = owner_refs + .iter() + .filter(|owner| is_supported_sandbox_owner_reference(owner)); let Some(owner) = sandbox_refs.next() else { + let unsupported_sandbox_api_versions = owner_refs + .iter() + .filter(|owner| owner.kind == SANDBOX_KIND) + .map(|owner| owner.api_version.as_str()) + .collect::>(); + if !unsupported_sandbox_api_versions.is_empty() { + warn!( + api_versions = ?unsupported_sandbox_api_versions, + supported_api_versions = ?[ + SANDBOX_API_VERSION_FULL_V1BETA1, + SANDBOX_API_VERSION_FULL_V1ALPHA1, + ], + "pod Sandbox ownerReference uses unsupported apiVersion" + ); + } return Err(Status::permission_denied( "pod is not controlled by an OpenShell Sandbox", )); @@ -394,11 +447,30 @@ fn sandbox_owner_reference(pod: &Pod) -> Result { )); } Ok(SandboxOwnerReference { + api_version: owner.api_version.clone(), name: owner.name.clone(), uid: owner.uid.clone(), }) } +fn is_supported_sandbox_owner_reference( + owner: &k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference, +) -> bool { + owner.kind == SANDBOX_KIND + && matches!( + owner.api_version.as_str(), + SANDBOX_API_VERSION_FULL_V1BETA1 | SANDBOX_API_VERSION_FULL_V1ALPHA1 + ) +} + +fn should_try_next_sandbox_api_version(err: &KubeError) -> bool { + // Kubernetes returns a structured 404 for some missing API resources and a + // raw "404 page not found" body for others. Both mean the probed + // group/version is unavailable and the next supported Sandbox API version + // should be tried. + matches!(err, KubeError::Api(api) if api.code == 404) +} + #[allow(clippy::result_large_err)] fn validate_sandbox_owner_reference( owner: &SandboxOwnerReference, @@ -486,6 +558,34 @@ mod tests { h } + fn kube_api_error(code: u16, message: &str) -> KubeError { + KubeError::Api(kube::core::ErrorResponse { + status: if code == 404 { + "404 Not Found".to_string() + } else { + "Failure".to_string() + }, + message: message.to_string(), + reason: "Failed to parse error data".to_string(), + code, + }) + } + + #[test] + fn sandbox_api_version_probe_retries_on_structured_and_raw_404() { + let structured = kube_api_error(404, "could not find the requested resource"); + assert!(should_try_next_sandbox_api_version(&structured)); + + let raw = kube_api_error(404, "404 page not found\n"); + assert!(should_try_next_sandbox_api_version(&raw)); + } + + #[test] + fn sandbox_api_version_probe_keeps_non_404_errors() { + let err = kube_api_error(403, "sandboxes.agents.x-k8s.io is forbidden"); + assert!(!should_try_next_sandbox_api_version(&err)); + } + fn token_review_status( authenticated: bool, audiences: Vec<&str>, @@ -515,8 +615,12 @@ mod tests { } fn sandbox_owner(name: &str, uid: &str) -> OwnerReference { + sandbox_owner_with_api_version(SANDBOX_API_VERSION_FULL_V1BETA1, name, uid) + } + + fn sandbox_owner_with_api_version(api_version: &str, name: &str, uid: &str) -> OwnerReference { OwnerReference { - api_version: SANDBOX_API_VERSION_FULL.to_string(), + api_version: api_version.to_string(), block_owner_deletion: None, controller: Some(true), kind: SANDBOX_KIND.to_string(), @@ -549,7 +653,7 @@ mod tests { fn sandbox_cr(name: &str, uid: &str, sandbox_id: &str) -> DynamicObject { let sandbox_gvk = - GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION, SANDBOX_KIND); + GroupVersionKind::gvk(SANDBOX_API_GROUP, SANDBOX_API_VERSION_V1BETA1, SANDBOX_KIND); let sandbox_resource = ApiResource::from_gvk(&sandbox_gvk); let mut cr = DynamicObject::new(name, &sandbox_resource); cr.metadata.uid = Some(uid.to_string()); @@ -681,6 +785,27 @@ mod tests { assert_eq!( owner, SandboxOwnerReference { + api_version: SANDBOX_API_VERSION_FULL_V1BETA1.to_string(), + name: "sandbox-a".to_string(), + uid: "cr-uid-a".to_string(), + } + ); + } + + #[test] + fn sandbox_owner_reference_accepts_v1alpha1_owner() { + let pod = pod_with_owner_refs(vec![sandbox_owner_with_api_version( + SANDBOX_API_VERSION_FULL_V1ALPHA1, + "sandbox-a", + "cr-uid-a", + )]); + + let owner = sandbox_owner_reference(&pod).expect("expected v1alpha1 Sandbox owner"); + + assert_eq!( + owner, + SandboxOwnerReference { + api_version: SANDBOX_API_VERSION_FULL_V1ALPHA1.to_string(), name: "sandbox-a".to_string(), uid: "cr-uid-a".to_string(), } @@ -696,6 +821,20 @@ mod tests { assert_eq!(err.code(), tonic::Code::PermissionDenied); } + #[test] + fn sandbox_owner_reference_rejects_unsupported_sandbox_api_version() { + let pod = pod_with_owner_refs(vec![sandbox_owner_with_api_version( + "agents.x-k8s.io/v1", + "sandbox-a", + "cr-uid-a", + )]); + + let err = + sandbox_owner_reference(&pod).expect_err("unsupported apiVersion must fail closed"); + + assert_eq!(err.code(), tonic::Code::PermissionDenied); + } + #[test] fn sandbox_owner_reference_requires_controlling_owner() { let mut owner = sandbox_owner("sandbox-a", "cr-uid-a"); @@ -722,6 +861,7 @@ mod tests { #[test] fn validate_sandbox_owner_reference_requires_matching_cr_uid_and_label() { let owner = SandboxOwnerReference { + api_version: SANDBOX_API_VERSION_FULL_V1BETA1.to_string(), name: "sandbox-a".to_string(), uid: "cr-uid-a".to_string(), }; diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 0a25eceeb5..5ab7865194 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -50,6 +50,10 @@ kubectl -n agent-sandbox-system get pods The controller pod should reach `Running` status within a few seconds. For cluster-specific setup instructions, including KinD and GKE walkthroughs, refer to the [Agent Sandbox getting started guide](https://agent-sandbox.sigs.k8s.io/docs/getting_started/). +### Upgrade Agent Sandbox + +OpenShell detects the served Agent Sandbox `Sandbox` API when the Kubernetes gateway first needs it and caches that choice for the gateway process. If you upgrade Agent Sandbox in place, restart the OpenShell gateway after the Agent Sandbox controller and CRD rollout completes so the gateway can detect the served API versions again. Existing sandboxes keep running during the upgrade, and the restarted gateway can continue managing them. + ## Install OpenShell diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 9e944715cf..341d9e9f46 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -308,7 +308,9 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | -The Kubernetes driver creates namespaced `agents.x-k8s.io/v1alpha1` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. The Agent Sandbox controller turns those resources into sandbox pods and related storage. +The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. + +If Agent Sandbox is upgraded in place, restart the OpenShell gateway after the controller and CRD rollout completes so the gateway can detect the served API versions again. `Sandbox.spec.volumeClaimTemplates` is immutable after creation. To change storage configuration, delete the sandbox and create a new one with the updated spec. diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index bea1c01d39..47b8730dc1 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -51,9 +51,10 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck source=e2e/support/gateway-common.sh source "${ROOT}/e2e/support/gateway-common.sh" -# Upstream agent-sandbox release. Bump in lockstep with the supported Sandbox -# field set in crates/openshell-driver-kubernetes (see sandbox_to_k8s_spec). -AGENT_SANDBOX_VERSION="${AGENT_SANDBOX_VERSION:-v0.4.6}" +# Upstream agent-sandbox release. The Kubernetes driver supports the v1beta1 +# Sandbox API introduced in v0.5.0 and falls back to v1alpha1 for v0.4.6 +# clusters. Override this env var to exercise the v1alpha1 controller release. +AGENT_SANDBOX_VERSION="${AGENT_SANDBOX_VERSION:-v0.5.0}" e2e_preserve_mise_dirs e2e_align_docker_host_with_cli_context @@ -88,6 +89,28 @@ kctl() { kubectl --context "${KUBE_CONTEXT}" "$@" } +wait_for_agent_sandbox_crd() { + local deadline + local established + + deadline=$(( $(date +%s) + 120 )) + while [ "$(date +%s)" -lt "${deadline}" ]; do + if kctl get crd/sandboxes.agents.x-k8s.io >/dev/null 2>&1; then + established="$(kctl get crd/sandboxes.agents.x-k8s.io \ + -o 'jsonpath={.status.conditions[?(@.type=="Established")].status}' \ + 2>/dev/null || true)" + if [ "${established}" = "True" ]; then + return 0 + fi + fi + sleep 2 + done + + echo "Timed out waiting for agent-sandbox Sandbox CRD to become Established" >&2 + kctl get crd/sandboxes.agents.x-k8s.io -o yaml >&2 || true + return 1 +} + helmctl() { helm --kube-context "${KUBE_CONTEXT}" "$@" } @@ -533,7 +556,7 @@ fi echo "Installing agent-sandbox CRDs and controller (${AGENT_SANDBOX_VERSION})..." _agent_sandbox_base="https://github.com/kubernetes-sigs/agent-sandbox/releases/download/${AGENT_SANDBOX_VERSION}" kctl apply -f "${_agent_sandbox_base}/manifest.yaml" -kctl wait --for=condition=Established crd/sandboxes.agents.x-k8s.io --timeout=120s +wait_for_agent_sandbox_crd kctl -n agent-sandbox-system rollout status deployment/agent-sandbox-controller --timeout=300s helm_extra_args=() diff --git a/tasks/scripts/helm-k3s-local.sh b/tasks/scripts/helm-k3s-local.sh index 59d8939a62..f9ac186f52 100755 --- a/tasks/scripts/helm-k3s-local.sh +++ b/tasks/scripts/helm-k3s-local.sh @@ -33,9 +33,10 @@ DEFAULT_SANDBOX_PRELOAD_IMAGE="ghcr.io/nvidia/openshell-community/sandboxes/base PRELOAD_SANDBOX_IMAGE="${HELM_K3S_PRELOAD_SANDBOX_IMAGE-${DEFAULT_SANDBOX_PRELOAD_IMAGE}}" # Upstream agent-sandbox release pinned for both CRDs/controller and extensions. -# Refresh this tag in lockstep with the supported field set in -# crates/openshell-driver-kubernetes (see sandbox_to_k8s_spec). -AGENT_SANDBOX_VERSION="${AGENT_SANDBOX_VERSION:-v0.4.6}" +# The Kubernetes driver supports the v1beta1 Sandbox API introduced in v0.5.0 +# and falls back to v1alpha1 for v0.4.6 clusters. Override this env var to +# exercise the v1alpha1 controller release. +AGENT_SANDBOX_VERSION="${AGENT_SANDBOX_VERSION:-v0.5.0}" default_kubeconfig="${ROOT}/kubeconfig" if [[ -n "${HELM_K3S_KUBECONFIG:-}" ]]; then diff --git a/tasks/test.toml b/tasks/test.toml index 444ea15e16..1d0f97856a 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -94,6 +94,18 @@ run = "e2e/rust/e2e-podman.sh" description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST= to scope to one test)" run = "e2e/rust/e2e-kubernetes.sh" +["e2e:kubernetes:v1alpha1"] +description = "Run Kubernetes e2e against Agent Sandbox v1alpha1" +env = { AGENT_SANDBOX_VERSION = "v0.4.6" } +run = "e2e/rust/e2e-kubernetes.sh" + +["e2e:kubernetes:agent-sandbox-versions"] +description = "Run Kubernetes e2e against Agent Sandbox v1beta1 and v1alpha1" +run = [ + "e2e/rust/e2e-kubernetes.sh", + "AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", +] + ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } From 45e5a5d1283e6514f90d6d9db7ea4f462b824e1d Mon Sep 17 00:00:00 2001 From: Gal Zaidman <38602062+Gal-Zaidman@users.noreply.github.com> Date: Fri, 26 Jun 2026 23:33:32 +0300 Subject: [PATCH 16/20] fix(server): prevent exec relays from hanging on idle connections (#1992) Add HTTP/2 keepalive on supervisor multiplex connections so half-dead sessions cannot leave in-flight exec relays parked indefinitely. Configure SSH keepalive on exec relay clients so long silent commands are not timed out on stdout idle alone; wedged or orphaned relays fail after missed keepalives instead. After a command reports exit status, bound how long the gateway waits for the trailing channel close. Return UNAVAILABLE when a relay closes before reporting exit status rather than defaulting to exit code 1. Signed-off-by: Gal Zaidman --- architecture/gateway.md | 9 +++ crates/openshell-server/src/grpc/sandbox.rs | 66 +++++++++++++++++++-- crates/openshell-server/src/multiplex.rs | 13 +++- 3 files changed, 80 insertions(+), 8 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index 2e02562491..d873b2a105 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -368,6 +368,15 @@ The same relay pattern backs interactive SSH, command execution, file sync, and local service forwarding. The gateway tracks live sessions in memory and persists session records so tokens can expire or be revoked. +Relay liveness has two backstops so a reset supervisor session cannot leave a +request parked forever. The gateway runs server-side HTTP/2 keepalive on +supervisor connections, and each exec relay's SSH client uses SSH keepalive: an +exec channel may be legitimately silent for a long time (e.g. an agent whose +stdout is redirected to a file), so the exec is never ended on output-idle +alone — instead an unanswered keepalive on a wedged or orphaned relay closes the +channel and returns the exec with an error. Once a command reports its exit +status, the gateway also bounds how long it waits for the trailing channel close. + `ForwardTcp` is the client-facing byte stream for SSH and service forwarding. The first frame is a `TcpForwardInit` that carries the sandbox ID, an authorization token from `CreateSshSession`, and an explicit target: diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 1570b8e034..04d5a4ed52 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -1485,6 +1485,36 @@ fn shell_escape(value: &str) -> Result { /// Maximum total length of the assembled shell command string. const MAX_COMMAND_STRING_LEN: usize = 256 * 1024; // 256 KiB +/// SSH keepalive for silent exec relays; stdout idle is not a timeout signal. +const EXEC_KEEPALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(15); + +/// Allow this many missed keepalive responses before russh fails the relay. +const EXEC_KEEPALIVE_MAX: usize = 4; + +/// Max wait for a trailing `Close` after `ExitStatus`. +const EXEC_POST_EXIT_CLOSE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); + +/// russh client config for exec relays. +fn exec_ssh_client_config() -> russh::client::Config { + russh::client::Config { + keepalive_interval: Some(EXEC_KEEPALIVE_INTERVAL), + keepalive_max: EXEC_KEEPALIVE_MAX, + ..Default::default() + } +} + +/// Treat channel EOF before an exit status as relay failure, not exit code 1. +fn exec_loop_result(exit_code: Option) -> Result { + exit_code.map_or_else( + || { + Err(Status::unavailable( + "exec relay closed before the command reported an exit status", + )) + }, + Ok, + ) +} + fn build_remote_exec_command(req: &ExecSandboxRequest) -> Result { let mut parts = Vec::new(); let mut env_entries = req.environment.iter().collect::>(); @@ -1691,7 +1721,7 @@ async fn run_interactive_exec_with_russh( .await .map_err(|e| Status::internal(format!("failed to connect to ssh proxy: {e}")))?; - let config = Arc::new(russh::client::Config::default()); + let config = Arc::new(exec_ssh_client_config()); let mut client = russh::client::connect_stream(config, stream, SandboxSshClientHandler) .await .map_err(|e| Status::internal(format!("failed to establish ssh transport: {e}")))?; @@ -1747,7 +1777,19 @@ async fn run_interactive_exec_with_russh( }); let mut exit_code: Option = None; - while let Some(msg) = read_half.wait().await { + loop { + // Bound the post-ExitStatus wait against a lost Close. + let msg = if exit_code.is_some() { + match tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, read_half.wait()).await { + Ok(Some(msg)) => msg, + Ok(None) | Err(_) => break, + } + } else { + match read_half.wait().await { + Some(msg) => msg, + None => break, + } + }; match msg { ChannelMsg::Data { data } => { let event = Ok(ExecSandboxEvent { @@ -1788,7 +1830,7 @@ async fn run_interactive_exec_with_russh( .disconnect(russh::Disconnect::ByApplication, "exec complete", "en") .await; - Ok(exit_code.unwrap_or(1)) + exec_loop_result(exit_code) } /// Create a localhost SSH proxy that bridges to a relay `DuplexStream`. @@ -1850,7 +1892,7 @@ async fn run_exec_with_russh( .await .map_err(|e| Status::internal(format!("failed to connect to ssh proxy: {e}")))?; - let config = Arc::new(russh::client::Config::default()); + let config = Arc::new(exec_ssh_client_config()); let mut client = russh::client::connect_stream(config, stream, SandboxSshClientHandler) .await .map_err(|e| Status::internal(format!("failed to establish ssh transport: {e}")))?; @@ -1898,7 +1940,19 @@ async fn run_exec_with_russh( .map_err(|e| Status::internal(format!("failed to close ssh stdin: {e}")))?; let mut exit_code: Option = None; - while let Some(msg) = channel.wait().await { + loop { + // Bound the post-ExitStatus wait against a lost Close. + let msg = if exit_code.is_some() { + match tokio::time::timeout(EXEC_POST_EXIT_CLOSE_TIMEOUT, channel.wait()).await { + Ok(Some(msg)) => msg, + Ok(None) | Err(_) => break, + } + } else { + match channel.wait().await { + Some(msg) => msg, + None => break, + } + }; match msg { ChannelMsg::Data { data } => { let _ = tx @@ -1936,7 +1990,7 @@ async fn run_exec_with_russh( .disconnect(russh::Disconnect::ByApplication, "exec complete", "en") .await; - Ok(exit_code.unwrap_or(1)) + exec_loop_result(exit_code) } // --------------------------------------------------------------------------- diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index f4faa0867f..9e70c64721 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -12,7 +12,7 @@ use http_body::Body; use http_body_util::BodyExt; use hyper::body::Incoming; use hyper_util::{ - rt::{TokioExecutor, TokioIo}, + rt::{TokioExecutor, TokioIo, TokioTimer}, server::conn::auto::Builder, service::TowerToHyperService, }; @@ -185,7 +185,16 @@ impl MultiplexService { let service = MultiplexedService::new(grpc_service, http_service); let mut builder = Builder::new(TokioExecutor::new()); - builder.http2().adaptive_window(true); + // Server-side HTTP/2 keepalive: supervisors hold long-lived sessions, and without + // it the gateway never PINGs them, so idle/half-dead connections linger and orphan + // in-flight relay execs. The timer is required — hyper panics on the keepalive + // interval without one. + builder + .http2() + .timer(TokioTimer::new()) + .adaptive_window(true) + .keep_alive_interval(Some(Duration::from_secs(20))) + .keep_alive_timeout(Duration::from_secs(10)); builder .serve_connection_with_upgrades(TokioIo::new(stream), service) From 7e0cce405d1954f0945aab4e48098ee733c0e2c0 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 26 Jun 2026 13:35:54 -0700 Subject: [PATCH 17/20] fix(build): use zig archive tools for cross builds (#2014) Signed-off-by: Taylor Mutch --- tasks/scripts/setup-zig-cc-wrapper.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tasks/scripts/setup-zig-cc-wrapper.sh b/tasks/scripts/setup-zig-cc-wrapper.sh index ea78f5d284..c21e0a3487 100755 --- a/tasks/scripts/setup-zig-cc-wrapper.sh +++ b/tasks/scripts/setup-zig-cc-wrapper.sh @@ -66,6 +66,22 @@ EOF chmod +x "$wrapper_dir/$tool" done +cat >"$wrapper_dir/ar" <"$wrapper_dir/ranlib" <"$toolchain_file" <>"$GITHUB_ENV" else echo "export CC_${target_env}=$wrapper_dir/cc" echo "export CXX_${target_env}=$wrapper_dir/c++" + echo "export AR_${target_env}=$wrapper_dir/ar" + echo "export RANLIB_${target_env}=$wrapper_dir/ranlib" echo "export CMAKE_TOOLCHAIN_FILE_${target_env}=$toolchain_file" if [[ $bare_target_env != "$target_env" ]]; then echo "export CC_${bare_target_env}=$wrapper_dir/cc" echo "export CXX_${bare_target_env}=$wrapper_dir/c++" + echo "export AR_${bare_target_env}=$wrapper_dir/ar" + echo "export RANLIB_${bare_target_env}=$wrapper_dir/ranlib" echo "export CMAKE_TOOLCHAIN_FILE_${bare_target_env}=$toolchain_file" fi fi From 8c784599c1ff12a0481dcad3efc8120572ad3acf Mon Sep 17 00:00:00 2001 From: Max Dubrinsky Date: Fri, 26 Jun 2026 18:23:33 -0400 Subject: [PATCH 18/20] fix(python): include generated proto stubs in Linux wheels (#2029) The generated protobuf/gRPC modules under python/openshell/_proto/ are gitignored. maturin honors .gitignore when collecting python-source files from a git checkout, so native builds (Linux release CI, local pip install .) dropped these modules and shipped an unimportable wheel. The macOS wheel only included them by accident: it builds in a Docker context whose COPY omits .git, so maturin had no gitignore to apply. Pin the stubs back in with explicit, package-relative [tool.maturin].include globs so inclusion no longer depends on whether .git is present in the build context. Add a wheel smoke check to the release-tag and release-dev workflows that installs each Linux wheel in a clean python:3.12-slim image and imports openshell.sandbox, and document the invariant in architecture/build.md. Closes #1705 Signed-off-by: Max Dubrinsky --- .github/workflows/release-dev.yml | 33 ++++++++++++++++++++++++++++- .github/workflows/release-tag.yml | 35 ++++++++++++++++++++++++++++++- architecture/build.md | 10 +++++++++ pyproject.toml | 7 +++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index dbf28dce3e..6d8c4d2a54 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -679,7 +679,7 @@ jobs: smoke-linux-dev-artifacts: name: Smoke Linux Dev Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] timeout-minutes: 20 strategy: fail-fast: false @@ -709,6 +709,18 @@ jobs: kind: rpm artifact_arch: arm64 rpm_arch: aarch64 + - name: python-wheel-amd64 + runner: linux-amd64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: amd64 + rpm_arch: x86_64 + - name: python-wheel-arm64 + runner: linux-arm64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: arm64 + rpm_arch: aarch64 runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -743,6 +755,25 @@ jobs: dnf install -y ./package-input/openshell-[0-9]*.rpm ./package-input/openshell-gateway-*.rpm LD_BIND_NOW=1 openshell-gateway --version + - name: Download Python wheel artifact + if: matrix.kind == 'wheel' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-wheels-linux-${{ matrix.artifact_arch }} + path: wheel-input/ + + - name: Smoke Python wheel + if: matrix.kind == 'wheel' + run: | + set -euo pipefail + pip install --no-cache-dir wheel-input/*.whl + python - <<'PY' + import importlib + for name in ("openshell", "openshell._proto", "openshell.sandbox"): + importlib.import_module(name) + print(name, "OK") + PY + # --------------------------------------------------------------------------- # Create / update the dev GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 02cf0ee193..3ab070fd8e 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -715,7 +715,7 @@ jobs: smoke-linux-release-artifacts: name: Smoke Linux Release Artifacts (${{ matrix.name }}) - needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm] + needs: [build-gateway-binary-linux, build-driver-vm-linux, build-deb, build-rpm, build-python-wheels-linux] timeout-minutes: 20 strategy: fail-fast: false @@ -763,6 +763,20 @@ jobs: artifact_arch: arm64 rpm_arch: aarch64 target: aarch64-unknown-linux-gnu + - name: python-wheel + runner: linux-amd64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: amd64 + rpm_arch: x86_64 + target: x86_64-unknown-linux-gnu + - name: python-wheel-arm64 + runner: linux-arm64-cpu8 + image: python:3.12-slim + kind: wheel + artifact_arch: arm64 + rpm_arch: aarch64 + target: aarch64-unknown-linux-gnu runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} @@ -822,6 +836,25 @@ jobs: dnf install -y ./package-input/openshell-[0-9]*.rpm ./package-input/openshell-gateway-*.rpm LD_BIND_NOW=1 openshell-gateway --version + - name: Download Python wheel artifact + if: matrix.kind == 'wheel' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: python-wheels-linux-${{ matrix.artifact_arch }} + path: wheel-input/ + + - name: Smoke Python wheel + if: matrix.kind == 'wheel' + run: | + set -euo pipefail + pip install --no-cache-dir wheel-input/*.whl + python - <<'PY' + import importlib + for name in ("openshell", "openshell._proto", "openshell.sandbox"): + importlib.import_module(name) + print(name, "OK") + PY + # --------------------------------------------------------------------------- # Create a tagged GitHub Release with CLI, gateway, driver, and wheels # --------------------------------------------------------------------------- diff --git a/architecture/build.md b/architecture/build.md index 6cd7b15d28..a3cb2e25fb 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -126,6 +126,16 @@ contexts use `KIND_EXPERIMENTAL_PROVIDER=docker|podman` when set, and ambiguous or unknown contexts require an explicit `CONTAINER_ENGINE`. Other image builds do not infer from kube context. +## Python Wheel Packaging + +The generated protobuf/gRPC stubs under `python/openshell/_proto/` are gitignored +build outputs of `mise run python:proto`. maturin honors `.gitignore` when +collecting `python-source` files, so native builds (Linux CI, local +`pip install .`) would drop them and ship an unimportable wheel. `pyproject.toml` +pins them back in with `[tool.maturin].include` globs. The release workflows +install each Linux wheel in a clean image and import `openshell.sandbox` as a +smoke check. + ## CI and E2E Required checks run on GitHub Actions. Workflows that use NVIDIA self-hosted runners trigger from copy-pr-bot mirror branches, so trusted PRs are mirrored into `pull-request/` branches before those workflows run. diff --git a/pyproject.toml b/pyproject.toml index b45081e3d1..dab81c57f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,13 @@ bindings = "bin" manifest-path = "crates/openshell-cli/Cargo.toml" python-source = "python" module-name = "openshell" +# _proto/ stubs are gitignored; force them into the wheel so git-aware +# maturin builds don't drop them (package-relative paths). +include = [ + "openshell/_proto/*_pb2.py", + "openshell/_proto/*_pb2_grpc.py", + "openshell/_proto/*.pyi", +] [tool.ruff] target-version = "py312" From 7bce1223dcd61fdc91969c6a3cef425fd95fc5dc Mon Sep 17 00:00:00 2001 From: krishicks Date: Fri, 26 Jun 2026 15:52:16 -0700 Subject: [PATCH 19/20] feat(policy): add JSON-RPC and MCP L7 policies (#1865) Add policy schema, proto, provider profile, OPA, and L7 proxy support for `protocol: json-rpc` and `protocol: mcp`. Generic JSON-RPC endpoints match exact method names only, with `method: "*"` as the all-method sentinel; wildcard/glob methods and params matchers are rejected. Parse JSON-RPC request bodies and batches in the forward proxy, deny response-shaped client frames, limit receive-stream GET allowance to MCP endpoints, and redact params in decision logs. Preserve L7 rule params on the proto load path so MCP `tools/call` tool filters behave like YAML-loaded policies. Add MCP conformance coverage, JSON-RPC L7 e2e coverage, and docs for the new protocols and current matcher limitations. Signed-off-by: Kris Hicks Co-authored-by: ddurst <267424412+ddurst-nvidia@users.noreply.github.com> --- .github/workflows/e2e-test.yml | 15 + Cargo.lock | 13 + Cargo.toml | 3 +- architecture/sandbox.md | 11 + crates/openshell-cli/src/policy_update.rs | 2 + crates/openshell-policy/src/lib.rs | 773 ++++++-- crates/openshell-policy/src/merge.rs | 2 + crates/openshell-providers/src/profiles.rs | 80 +- .../src/mechanistic_mapper.rs | 1 + crates/openshell-server/src/grpc/policy.rs | 3 + .../openshell-supervisor-network/Cargo.toml | 1 + .../data/sandbox-policy.rego | 183 ++ .../src/l7/graphql.rs | 1 + .../src/l7/http.rs | 199 ++ .../src/l7/jsonrpc.rs | 750 ++++++++ .../src/l7/mod.rs | 1616 +++++++++++++---- .../src/l7/relay.rs | 1059 ++++++++++- .../src/l7/rest.rs | 169 +- .../src/l7/websocket.rs | 3 + .../openshell-supervisor-network/src/opa.rs | 1302 ++++++++++++- .../src/policy_local.rs | 4 + .../openshell-supervisor-network/src/proxy.rs | 204 ++- docs/reference/policy-schema.mdx | 119 +- docs/sandboxes/policies.mdx | 53 +- e2e/mcp-conformance.sh | 519 ++++++ e2e/mcp-conformance/Dockerfile.client | 26 + .../Dockerfile.client.dockerignore | 7 + e2e/mcp-conformance/README.md | 82 + .../client-through-openshell.sh | 100 + e2e/mcp-conformance/expected-failures.yml | 9 + e2e/mcp-conformance/host-bridge.py | 249 +++ e2e/mcp-conformance/policy-template.yaml | 56 + e2e/mcp-conformance/render-policy.py | 75 + e2e/mcp-conformance/runner-shim.mjs | 139 ++ e2e/rust/Cargo.toml | 5 + e2e/rust/tests/forward_proxy_jsonrpc_l7.rs | 520 ++++++ e2e/with-docker-gateway.sh | 2 +- mise.toml | 2 +- proto/sandbox.proto | 51 +- tasks/test.toml | 12 +- 40 files changed, 7841 insertions(+), 579 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/l7/http.rs create mode 100644 crates/openshell-supervisor-network/src/l7/jsonrpc.rs create mode 100755 e2e/mcp-conformance.sh create mode 100644 e2e/mcp-conformance/Dockerfile.client create mode 100644 e2e/mcp-conformance/Dockerfile.client.dockerignore create mode 100644 e2e/mcp-conformance/README.md create mode 100755 e2e/mcp-conformance/client-through-openshell.sh create mode 100644 e2e/mcp-conformance/expected-failures.yml create mode 100644 e2e/mcp-conformance/host-bridge.py create mode 100644 e2e/mcp-conformance/policy-template.yaml create mode 100644 e2e/mcp-conformance/render-policy.py create mode 100644 e2e/mcp-conformance/runner-shim.mjs create mode 100644 e2e/rust/tests/forward_proxy_jsonrpc_l7.rs diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 2950ab36ef..8815db637a 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -44,6 +44,9 @@ jobs: cmd: "mise run --no-deps --skip-deps e2e:podman:rootless" apt_packages: "openssh-client podman uidmap" rootless: true + - suite: mcp + cmd: "mise run --no-deps --skip-deps e2e:mcp" + apt_packages: "" container: image: ghcr.io/nvidia/openshell/ci:latest credentials: @@ -65,6 +68,17 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs['checkout-ref'] || github.sha }} + persist-credentials: false + + - name: Check out MCP conformance tests + if: matrix.suite == 'mcp' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + repository: modelcontextprotocol/conformance + # Pin after v0.1.16 to include the tools_call client scenario fix. + ref: b9041ea41b0188581803459dbae71bc7e02fd995 + path: .cache/mcp-conformance + persist-credentials: false - name: Install OS test dependencies if: matrix.apt_packages != '' @@ -104,6 +118,7 @@ jobs: - name: Run tests env: OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }} + OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE: ${{ format('openshell-mcp-conformance-client:{0}', inputs.image-tag) }} E2E_CMD: ${{ matrix.cmd }} run: | if [ "${{ matrix.rootless }}" = "true" ]; then diff --git a/Cargo.lock b/Cargo.lock index 672fd71ab3..d900b67a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3946,6 +3946,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-tungstenite 0.26.2", + "tower-mcp-types", "tracing", "uuid", "webpki-roots 1.0.7", @@ -6630,6 +6631,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" +[[package]] +name = "tower-mcp-types" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6511f1f32c7cb7fd4525edc0eb4dcf307db8f7eceb2833ab24a37b4cc10cda61" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "tower-service" version = "0.3.3" diff --git a/Cargo.toml b/Cargo.toml index 86025646a9..f450cd5c8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ members = ["crates/*"] [workspace.package] version = "0.0.0" edition = "2024" -rust-version = "1.88" +rust-version = "1.90" license = "Apache-2.0" repository = "https://github.com/NVIDIA/OpenShell" @@ -73,6 +73,7 @@ serde_json = "1" serde_yml = "0.0.12" toml = "0.8" apollo-parser = "0.8.5" +tower-mcp-types = "0.12.0" # HTTP client reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots"] } diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 903af3a4d2..31a357abf9 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -49,6 +49,17 @@ paths, such as proxy support files or GPU device paths when a GPU is present. All ordinary agent egress is routed through the sandbox proxy. The proxy identifies the calling binary, checks trust-on-first-use binary identity, rejects unsafe internal destinations, and evaluates the active policy. +For inspected HTTP traffic, the proxy can enforce REST method/path rules, +WebSocket upgrade and text-message rules, GraphQL operation rules, and +MCP method, tool, and supported params rules or generic JSON-RPC method rules +on sandbox-to-server request bodies. MCP and JSON-RPC inspection buffers up to +the endpoint `mcp.max_body_bytes` or `json_rpc.max_body_bytes` limit. MCP +`tools/call` tool names are checked against the spec-recommended syntax by +default before policy evaluation, with a per-endpoint `mcp.strict_tool_names` +compatibility opt-out. Generic JSON-RPC policies do not support `params` +matchers; generic JSON-RPC rules match only the method. +JSON-RPC responses and server-to-client MCP messages on response or SSE streams +are relayed but are not currently parsed for policy enforcement. `https://inference.local` is special. It bypasses OPA network policy and is handled by the inference interception path: diff --git a/crates/openshell-cli/src/policy_update.rs b/crates/openshell-cli/src/policy_update.rs index 57656b878f..1f1f647506 100644 --- a/crates/openshell-cli/src/policy_update.rs +++ b/crates/openshell-cli/src/policy_update.rs @@ -205,6 +205,7 @@ fn group_allow_rules(specs: &[String]) -> Result Result, + #[serde(default, skip_serializing_if = "Option::is_none")] + mcp: Option, } // Signature dictated by serde's `skip_serializing_if`, which requires `&T`. @@ -158,6 +162,109 @@ fn is_zero_u32(v: &u32) -> bool { *v == 0 } +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct JsonRpcConfigDef { + #[serde(default, skip_serializing_if = "is_zero_u32")] + max_body_bytes: u32, +} + +fn json_rpc_config_from_proto(max_body_bytes: u32) -> Option { + (max_body_bytes > 0).then_some(JsonRpcConfigDef { max_body_bytes }) +} + +// MCP rides the same HTTP/JSON-RPC inspection machinery at runtime, but it +// gets its own policy stanza so user-authored YAML can name the primary +// protocol instead of treating MCP as generic JSON-RPC. +#[derive(Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct McpConfigDef { + #[serde(default, skip_serializing_if = "is_zero_u32")] + max_body_bytes: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + strict_tool_names: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + allow_all_known_mcp_methods: Option, +} + +fn mcp_config_from_proto(max_body_bytes: u32, mcp: Option<&McpOptions>) -> Option { + let strict_tool_names = mcp.and_then(|config| config.strict_tool_names); + let allow_all_known_mcp_methods = mcp.and_then(|config| config.allow_all_known_mcp_methods); + (max_body_bytes > 0 || strict_tool_names.is_some() || allow_all_known_mcp_methods.is_some()) + .then_some(McpConfigDef { + max_body_bytes, + strict_tool_names, + allow_all_known_mcp_methods, + }) +} + +/// Nested L7 config stanzas accepted by the YAML policy schema. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum L7ConfigStanza { + JsonRpc, + Mcp, +} + +impl L7ConfigStanza { + pub const ALL: [Self; 2] = [Self::JsonRpc, Self::Mcp]; + + #[must_use] + pub const fn key(self) -> &'static str { + match self { + Self::JsonRpc => "json_rpc", + Self::Mcp => "mcp", + } + } +} + +/// Parse an L7 nested config stanza and return the flattened runtime fields +/// consumed by the supervisor policy engine. +/// +/// The stanza schema stays tied to this crate's canonical serde definitions, so +/// adding a new supported field requires updating this conversion next to the +/// type that parses it. +pub fn l7_config_alias_runtime_fields( + stanza: L7ConfigStanza, + value: serde_json::Value, +) -> Result> { + match stanza { + L7ConfigStanza::JsonRpc => { + let JsonRpcConfigDef { max_body_bytes } = serde_json::from_value(value) + .map_err(|error| miette::miette!("invalid json_rpc config: {error}"))?; + let mut fields = Vec::new(); + if max_body_bytes > 0 { + fields.push(("json_rpc_max_body_bytes", serde_json::json!(max_body_bytes))); + } + Ok(fields) + } + L7ConfigStanza::Mcp => { + let McpConfigDef { + max_body_bytes, + strict_tool_names, + allow_all_known_mcp_methods, + } = serde_json::from_value(value) + .map_err(|error| miette::miette!("invalid mcp config: {error}"))?; + let mut fields = Vec::new(); + if max_body_bytes > 0 { + fields.push(("json_rpc_max_body_bytes", serde_json::json!(max_body_bytes))); + } + if let Some(strict_tool_names) = strict_tool_names { + fields.push(( + "mcp_strict_tool_names", + serde_json::json!(strict_tool_names), + )); + } + if let Some(allow_all_known_mcp_methods) = allow_all_known_mcp_methods { + fields.push(( + "mcp_allow_all_known_mcp_methods", + serde_json::json!(allow_all_known_mcp_methods), + )); + } + Ok(fields) + } + } +} + #[derive(Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct GraphqlOperationDef { @@ -192,16 +299,31 @@ struct L7AllowDef { operation_name: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] fields: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + tool: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + params: BTreeMap, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(untagged)] enum QueryMatcherDef { + // Short form: `query: { repo: "NVIDIA/*" }`. Glob(String), + // Expanded form: `query: { repo: { any: ["NVIDIA/*", "openai/*"] } }`. Any(QueryAnyDef), } -#[derive(Debug, Serialize, Deserialize)] +// MCP params can be authored as nested maps in YAML, but the runtime matcher +// map remains flat so the Rego policy can share query-param matching. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +enum ParamMatcherDef { + Matcher(QueryMatcherDef), + Object(BTreeMap), +} + +#[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct QueryAnyDef { #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -225,6 +347,10 @@ struct L7DenyRuleDef { operation_name: String, #[serde(default, skip_serializing_if = "Vec::is_empty")] fields: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + tool: Option, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + params: BTreeMap, } #[derive(Debug, Serialize, Deserialize)] @@ -241,6 +367,310 @@ struct NetworkBinaryDef { // YAML → proto conversion // --------------------------------------------------------------------------- +fn matcher_def_to_proto(matcher: QueryMatcherDef) -> L7QueryMatcher { + match matcher { + QueryMatcherDef::Glob(glob) => L7QueryMatcher { glob, any: vec![] }, + QueryMatcherDef::Any(any) => L7QueryMatcher { + glob: String::new(), + any: any.any, + }, + } +} + +fn matcher_proto_to_def(matcher: L7QueryMatcher) -> QueryMatcherDef { + if matcher.any.is_empty() { + QueryMatcherDef::Glob(matcher.glob) + } else { + QueryMatcherDef::Any(QueryAnyDef { any: matcher.any }) + } +} + +// Convert MCP params maps into the flat proto/Rego keyspace. Only `name` is +// currently enforced for tools/call, but this keeps the YAML shape compatible +// with any future MCP-owned params selectors. +fn flatten_param_matchers( + params: BTreeMap, +) -> BTreeMap { + let mut flattened = BTreeMap::new(); + for (key, matcher) in params { + flatten_param_matcher(&key, matcher, &mut flattened); + } + flattened +} + +// Walk one params subtree, carrying the flattened dot-path key accumulated so +// far. Leaf matchers are inserted into the map consumed by the runtime policy. +fn flatten_param_matcher( + key: &str, + matcher: ParamMatcherDef, + out: &mut BTreeMap, +) { + match matcher { + ParamMatcherDef::Matcher(matcher) => { + out.insert(key.to_string(), matcher); + } + ParamMatcherDef::Object(children) => { + for (child_key, child) in children { + let nested_key = format!("{key}.{child_key}"); + flatten_param_matcher(&nested_key, child, out); + } + } + } +} + +// Convert flat runtime params back to YAML. MCP gets readable nested params +// when the flat keys can be losslessly split. Non-MCP protocols keep flat keys +// only for lossless serialization; generic JSON-RPC validation rejects params +// matchers before enforcement. +fn flat_params_to_def( + protocol: &str, + params: BTreeMap, +) -> BTreeMap { + let flat = params.into_iter().collect::>(); + // MCP uses nested YAML for readability. Non-MCP protocols keep the flat + // form for lossless serialization of existing proto data only. + if !is_mcp_protocol(protocol) { + return flat_param_matchers_to_def(flat); + } + + let mut nested = BTreeMap::new(); + for (key, matcher) in &flat { + if insert_nested_param(&mut nested, key, ParamMatcherDef::Matcher(matcher.clone())).is_err() + { + return flat_param_matchers_to_def(flat); + } + } + nested +} + +fn flat_param_matchers_to_def( + params: Vec<(String, QueryMatcherDef)>, +) -> BTreeMap { + params + .into_iter() + .map(|(key, matcher)| (key, ParamMatcherDef::Matcher(matcher))) + .collect() +} + +// Build one nested params path from a flat key. Collisions such as `a` and +// `a.b` cannot round-trip as nested YAML, so callers fall back to the flat map. +fn insert_nested_param( + root: &mut BTreeMap, + key: &str, + matcher: ParamMatcherDef, +) -> Result<(), ()> { + let mut parts = key.split('.').peekable(); + let Some(first) = parts.next() else { + return Err(()); + }; + + if parts.peek().is_none() { + root.insert(first.to_string(), matcher); + return Ok(()); + } + + let child = root + .entry(first.to_string()) + .or_insert_with(|| ParamMatcherDef::Object(BTreeMap::new())); + let ParamMatcherDef::Object(children) = child else { + return Err(()); + }; + let remainder = parts.collect::>().join("."); + insert_nested_param(children, &remainder, matcher) +} + +// MCP `tool` is a policy convenience for the standard `tools/call` params.name +// field. When the endpoint method profile is enabled, authored tool selectors +// can omit method and are normalized to tools/call internally. Tool arguments +// intentionally have no policy matcher yet, so every allowed tool call permits +// all argument payloads by default. +fn params_with_tool( + mut params: BTreeMap, + tool: Option, +) -> BTreeMap { + if let Some(tool) = tool { + params + .entry("name".to_string()) + .or_insert_with(|| ParamMatcherDef::Matcher(tool)); + } + params +} + +fn allow_def_to_proto(_protocol: &str, allow: L7AllowDef) -> L7Allow { + let params = flatten_param_matchers(params_with_tool(allow.params, allow.tool)); + L7Allow { + method: allow.method, + path: allow.path, + command: allow.command, + operation_type: allow.operation_type, + operation_name: allow.operation_name, + fields: allow.fields, + query: allow + .query + .into_iter() + .map(|(key, matcher)| (key, matcher_def_to_proto(matcher))) + .collect(), + params: params + .into_iter() + .map(|(key, matcher)| (key, matcher_def_to_proto(matcher))) + .collect(), + } +} + +fn deny_def_to_proto(_protocol: &str, deny: L7DenyRuleDef) -> L7DenyRule { + let params = flatten_param_matchers(params_with_tool(deny.params, deny.tool)); + L7DenyRule { + method: deny.method, + path: deny.path, + command: deny.command, + operation_type: deny.operation_type, + operation_name: deny.operation_name, + fields: deny.fields, + query: deny + .query + .into_iter() + .map(|(key, matcher)| (key, matcher_def_to_proto(matcher))) + .collect(), + params: params + .into_iter() + .map(|(key, matcher)| (key, matcher_def_to_proto(matcher))) + .collect(), + } +} + +fn json_rpc_max_body_bytes(json_rpc: &Option, mcp: &Option) -> u32 { + // The proto has one JSON-RPC-family body limit. Prefer the MCP stanza when + // present because MCP policies should not need a shadow `json_rpc` block. + mcp.as_ref().map_or_else( + || json_rpc.as_ref().map_or(0, |config| config.max_body_bytes), + |config| config.max_body_bytes, + ) +} + +fn mcp_strict_tool_names(mcp: &Option) -> Option { + mcp.as_ref().and_then(|config| config.strict_tool_names) +} + +fn mcp_allow_all_known_mcp_methods(mcp: &Option) -> Option { + mcp.as_ref() + .and_then(|config| config.allow_all_known_mcp_methods) +} + +fn mcp_options(mcp: &Option) -> Option { + let strict_tool_names = mcp_strict_tool_names(mcp); + let allow_all_known_mcp_methods = mcp_allow_all_known_mcp_methods(mcp); + (strict_tool_names.is_some() || allow_all_known_mcp_methods.is_some()).then_some(McpOptions { + strict_tool_names, + allow_all_known_mcp_methods, + }) +} + +fn is_mcp_protocol(protocol: &str) -> bool { + protocol.eq_ignore_ascii_case("mcp") +} + +fn split_tool_param( + protocol: &str, + params: BTreeMap, +) -> (Option, BTreeMap) { + // Only MCP has the tool-name convention. Non-MCP protocols preserve proto + // params on round-trip without inventing MCP semantics. + if !is_mcp_protocol(protocol) { + return (None, params); + } + + let mut params = params; + let tool = params.remove("name"); + (tool, params) +} + +fn allow_proto_to_def( + protocol: &str, + allow: L7Allow, + mcp_allow_all_known_mcp_methods: bool, +) -> L7AllowDef { + let params: BTreeMap = allow + .params + .into_iter() + .map(|(key, matcher)| (key, matcher_proto_to_def(matcher))) + .collect(); + let (tool, params) = split_tool_param(protocol, params); + let params = flat_params_to_def(protocol, params); + let method = yaml_mcp_method( + protocol, + &allow.method, + tool.is_some(), + mcp_allow_all_known_mcp_methods, + ); + L7AllowDef { + method, + path: allow.path, + command: allow.command, + query: allow + .query + .into_iter() + .map(|(key, matcher)| (key, matcher_proto_to_def(matcher))) + .collect(), + operation_type: allow.operation_type, + operation_name: allow.operation_name, + fields: allow.fields, + tool, + params, + } +} + +fn deny_proto_to_def( + protocol: &str, + deny: &L7DenyRule, + mcp_allow_all_known_mcp_methods: bool, +) -> L7DenyRuleDef { + let params: BTreeMap = deny + .params + .iter() + .map(|(key, matcher)| (key.clone(), matcher_proto_to_def(matcher.clone()))) + .collect(); + let (tool, params) = split_tool_param(protocol, params); + let params = flat_params_to_def(protocol, params); + let method = yaml_mcp_method( + protocol, + &deny.method, + tool.is_some(), + mcp_allow_all_known_mcp_methods, + ); + L7DenyRuleDef { + method, + path: deny.path.clone(), + command: deny.command.clone(), + query: deny + .query + .iter() + .map(|(key, matcher)| (key.clone(), matcher_proto_to_def(matcher.clone()))) + .collect(), + operation_type: deny.operation_type.clone(), + operation_name: deny.operation_name.clone(), + fields: deny.fields.clone(), + tool, + params, + } +} + +fn yaml_mcp_method( + protocol: &str, + method: &str, + has_tool: bool, + mcp_allow_all_known_mcp_methods: bool, +) -> String { + if is_mcp_protocol(protocol) { + if !has_tool && method == "*" { + return String::new(); + } + if has_tool && method == "tools/call" && mcp_allow_all_known_mcp_methods { + return String::new(); + } + } + method.to_string() +} + fn to_proto(raw: PolicyFile) -> SandboxPolicy { let network_policies = raw .network_policies @@ -256,6 +686,9 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { .endpoints .into_iter() .map(|e| { + let protocol = e.protocol; + let allow_rules = e.rules; + let deny_rules = e.deny_rules; // Normalize port/ports: ports takes precedence, else // single port is promoted to ports array. let normalized_ports: Vec = if !e.ports.is_empty() { @@ -270,69 +703,20 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { path: e.path, port: normalized_ports.first().copied().unwrap_or(0), ports: normalized_ports, - protocol: e.protocol, + protocol: protocol.clone(), tls: e.tls, enforcement: e.enforcement, access: e.access, - rules: e - .rules + rules: allow_rules .into_iter() .map(|r| L7Rule { - allow: Some(L7Allow { - method: r.allow.method, - path: r.allow.path, - command: r.allow.command, - operation_type: r.allow.operation_type, - operation_name: r.allow.operation_name, - fields: r.allow.fields, - query: r - .allow - .query - .into_iter() - .map(|(key, matcher)| { - let proto = match matcher { - QueryMatcherDef::Glob(glob) => { - L7QueryMatcher { glob, any: vec![] } - } - QueryMatcherDef::Any(any) => L7QueryMatcher { - glob: String::new(), - any: any.any, - }, - }; - (key, proto) - }) - .collect(), - }), + allow: Some(allow_def_to_proto(&protocol, r.allow)), }) .collect(), allowed_ips: e.allowed_ips, - deny_rules: e - .deny_rules + deny_rules: deny_rules .into_iter() - .map(|d| L7DenyRule { - method: d.method, - path: d.path, - command: d.command, - operation_type: d.operation_type, - operation_name: d.operation_name, - fields: d.fields, - query: d - .query - .into_iter() - .map(|(key, matcher)| { - let proto = match matcher { - QueryMatcherDef::Glob(glob) => { - L7QueryMatcher { glob, any: vec![] } - } - QueryMatcherDef::Any(any) => L7QueryMatcher { - glob: String::new(), - any: any.any, - }, - }; - (key, proto) - }) - .collect(), - }) + .map(|deny| deny_def_to_proto(&protocol, deny)) .collect(), allow_encoded_slash: e.allow_encoded_slash, websocket_credential_rewrite: e.websocket_credential_rewrite, @@ -359,6 +743,8 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { credential_signing: e.credential_signing, signing_service: e.signing_service, signing_region: e.signing_region, + json_rpc_max_body_bytes: json_rpc_max_body_bytes(&e.json_rpc, &e.mcp), + mcp: mcp_options(&e.mcp), } }) .collect(), @@ -438,73 +824,50 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { } else { (clamp(e.ports.first().copied().unwrap_or(e.port)), vec![]) }; + let protocol = e.protocol.clone(); + let mcp_allow_all_known_mcp_methods = !is_mcp_protocol(&protocol) + || e.mcp + .as_ref() + .and_then(|options| options.allow_all_known_mcp_methods) + .unwrap_or(false); + let rules = e + .rules + .iter() + .map(|r| L7RuleDef { + allow: allow_proto_to_def( + &protocol, + r.allow.clone().unwrap_or_default(), + mcp_allow_all_known_mcp_methods, + ), + }) + .collect(); + let deny_rules: Vec = e + .deny_rules + .iter() + .map(|d| { + deny_proto_to_def(&protocol, d, mcp_allow_all_known_mcp_methods) + }) + .collect(); + let (json_rpc, mcp) = if is_mcp_protocol(&protocol) { + ( + None, + mcp_config_from_proto(e.json_rpc_max_body_bytes, e.mcp.as_ref()), + ) + } else { + (json_rpc_config_from_proto(e.json_rpc_max_body_bytes), None) + }; NetworkEndpointDef { host: e.host.clone(), path: e.path.clone(), port, ports, - protocol: e.protocol.clone(), + protocol, tls: e.tls.clone(), enforcement: e.enforcement.clone(), access: e.access.clone(), - rules: e - .rules - .iter() - .map(|r| { - let a = r.allow.clone().unwrap_or_default(); - L7RuleDef { - allow: L7AllowDef { - method: a.method, - path: a.path, - command: a.command, - operation_type: a.operation_type, - operation_name: a.operation_name, - fields: a.fields, - query: a - .query - .into_iter() - .map(|(key, matcher)| { - let yaml_matcher = if matcher.any.is_empty() { - QueryMatcherDef::Glob(matcher.glob) - } else { - QueryMatcherDef::Any(QueryAnyDef { - any: matcher.any, - }) - }; - (key, yaml_matcher) - }) - .collect(), - }, - } - }) - .collect(), + rules, allowed_ips: e.allowed_ips.clone(), - deny_rules: e - .deny_rules - .iter() - .map(|d| L7DenyRuleDef { - method: d.method.clone(), - path: d.path.clone(), - command: d.command.clone(), - operation_type: d.operation_type.clone(), - operation_name: d.operation_name.clone(), - fields: d.fields.clone(), - query: d - .query - .iter() - .map(|(key, matcher)| { - let yaml_matcher = if matcher.any.is_empty() { - QueryMatcherDef::Glob(matcher.glob.clone()) - } else { - QueryMatcherDef::Any(QueryAnyDef { - any: matcher.any.clone(), - }) - }; - (key.clone(), yaml_matcher) - }) - .collect(), - }) - .collect(), + deny_rules, allow_encoded_slash: e.allow_encoded_slash, websocket_credential_rewrite: e.websocket_credential_rewrite, request_body_credential_rewrite: e.request_body_credential_rewrite, @@ -527,6 +890,8 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { credential_signing: e.credential_signing.clone(), signing_service: e.signing_service.clone(), signing_region: e.signing_region.clone(), + json_rpc, + mcp, } }) .collect(), @@ -1164,6 +1529,35 @@ network_policies: assert!(parse_sandbox_policy(yaml).is_err()); } + #[test] + fn l7_config_stanza_runtime_fields_use_canonical_schema() { + let fields = l7_config_alias_runtime_fields( + L7ConfigStanza::Mcp, + serde_json::json!({ + "max_body_bytes": 131_072, + "strict_tool_names": false, + "allow_all_known_mcp_methods": true + }), + ) + .expect("valid mcp config"); + + assert_eq!( + fields, + vec![ + ("json_rpc_max_body_bytes", serde_json::json!(131_072)), + ("mcp_strict_tool_names", serde_json::json!(false)), + ("mcp_allow_all_known_mcp_methods", serde_json::json!(true)), + ] + ); + + let err = l7_config_alias_runtime_fields( + L7ConfigStanza::JsonRpc, + serde_json::json!({"on_parse_error": "allow"}), + ) + .expect_err("unknown JSON-RPC config fields must be rejected"); + assert!(err.to_string().contains("on_parse_error")); + } + #[test] fn ensure_sandbox_process_identity_fills_defaults() { let mut policy = restrictive_default_policy(); @@ -1934,6 +2328,155 @@ network_policies: assert_eq!(ep.deny_rules[0].fields, vec!["deleteRepository"]); } + #[test] + fn round_trip_preserves_json_rpc_max_body_bytes() { + let yaml = r" +version: 1 +network_policies: + jsonrpc_api: + name: jsonrpc_api + endpoints: + - host: jsonrpc.example.com + port: 443 + protocol: json-rpc + enforcement: enforce + json_rpc: + max_body_bytes: 131072 + rules: + - allow: + method: initialize + binaries: + - path: /usr/bin/curl +"; + let proto1 = parse_sandbox_policy(yaml).expect("parse failed"); + let yaml_out = serialize_sandbox_policy(&proto1).expect("serialize failed"); + let proto2 = parse_sandbox_policy(&yaml_out).expect("re-parse failed"); + + let ep = &proto2.network_policies["jsonrpc_api"].endpoints[0]; + assert_eq!(ep.protocol, "json-rpc"); + assert_eq!(ep.json_rpc_max_body_bytes, 131_072); + } + + #[test] + fn parse_mcp_rules_to_runtime_fields() { + let yaml = r" +version: 1 +network_policies: + mcp: + name: mcp + endpoints: + - host: mcp.example.com + port: 443 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + max_body_bytes: 131072 + strict_tool_names: false + rules: + - allow: + method: initialize + - allow: + method: tools/list + - allow: + method: tools/call + tool: + any: [search_web, list_tools] + deny_rules: + - method: tools/call + tool: send_email + binaries: + - path: /usr/bin/curl +"; + let proto = parse_sandbox_policy(yaml).expect("parse failed"); + let ep = &proto.network_policies["mcp"].endpoints[0]; + + assert_eq!(ep.protocol, "mcp"); + assert_eq!(ep.json_rpc_max_body_bytes, 131_072); + assert_eq!( + ep.mcp + .as_ref() + .and_then(|options| options.strict_tool_names), + Some(false) + ); + assert_eq!(ep.rules.len(), 3); + assert_eq!(ep.rules[2].allow.as_ref().unwrap().method, "tools/call"); + assert_eq!( + ep.rules[2].allow.as_ref().unwrap().params["name"].any, + vec!["search_web".to_string(), "list_tools".to_string()] + ); + assert_eq!(ep.deny_rules.len(), 1); + assert_eq!(ep.deny_rules[0].method, "tools/call"); + assert_eq!(ep.deny_rules[0].params["name"].glob, "send_email"); + } + + #[test] + fn round_trip_mcp_policy_serializes_mcp_expression() { + let yaml = r" +version: 1 +network_policies: + mcp: + name: mcp + endpoints: + - host: mcp.example.com + port: 443 + protocol: mcp + mcp: + max_body_bytes: 131072 + strict_tool_names: false + rules: + - allow: + method: tools/call + tool: search_web + deny_rules: + - method: tools/call + tool: + any: [send_email, delete_resource] + binaries: + - path: /usr/bin/curl +"; + let proto1 = parse_sandbox_policy(yaml).expect("parse failed"); + let yaml_out = serialize_sandbox_policy(&proto1).expect("serialize failed"); + let proto2 = parse_sandbox_policy(&yaml_out).expect("re-parse failed"); + + assert!(yaml_out.contains("protocol: mcp")); + assert!(yaml_out.contains("method: tools/call")); + assert!(yaml_out.contains("tool: search_web")); + assert!(yaml_out.contains("any:")); + assert!(yaml_out.contains("- send_email")); + assert!(yaml_out.contains("- delete_resource")); + assert!(yaml_out.contains("deny_rules:")); + assert!(!yaml_out.contains("arguments:")); + assert!(yaml_out.contains("mcp:")); + assert!(yaml_out.contains("strict_tool_names: false")); + assert_eq!(proto1, proto2); + } + + #[test] + fn parse_rejects_unsupported_json_rpc_config_fields() { + let yaml = r" +version: 1 +network_policies: + jsonrpc_api: + endpoints: + - host: jsonrpc.example.com + port: 443 + protocol: json-rpc + json_rpc: + max_body_bytes: 131072 + on_parse_error: deny + batch_policy: all + access: full + binaries: + - path: /usr/bin/curl +"; + + assert!( + parse_sandbox_policy(yaml).is_err(), + "unsupported json_rpc fields must not be silently accepted" + ); + } + #[test] fn round_trip_preserves_websocket_credential_rewrite() { let yaml = r" diff --git a/crates/openshell-policy/src/merge.rs b/crates/openshell-policy/src/merge.rs index 89c8bf8b36..04f390198d 100644 --- a/crates/openshell-policy/src/merge.rs +++ b/crates/openshell-policy/src/merge.rs @@ -749,6 +749,7 @@ fn expand_access_preset(protocol: &str, access: &str) -> Option> { operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::default(), }), }) .collect(), @@ -963,6 +964,7 @@ mod tests { operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::default(), }), } } diff --git a/crates/openshell-providers/src/profiles.rs b/crates/openshell-providers/src/profiles.rs index 530318fb72..2353c7e715 100644 --- a/crates/openshell-providers/src/profiles.rs +++ b/crates/openshell-providers/src/profiles.rs @@ -6,10 +6,10 @@ #![allow(deprecated)] // NetworkBinary::harness remains in the public proto for compatibility. use openshell_core::proto::{ - GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, NetworkBinary, NetworkEndpoint, - NetworkPolicyRule, ProviderCredentialRefresh, ProviderCredentialRefreshMaterial, - ProviderCredentialRefreshStrategy, ProviderProfile, ProviderProfileCategory, - ProviderProfileCredential, ProviderProfileDiscovery, + GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, McpOptions, NetworkBinary, + NetworkEndpoint, NetworkPolicyRule, ProviderCredentialRefresh, + ProviderCredentialRefreshMaterial, ProviderCredentialRefreshStrategy, ProviderProfile, + ProviderProfileCategory, ProviderProfileCredential, ProviderProfileDiscovery, }; use serde::ser::SerializeStruct; use serde::{Deserialize, Deserializer, Serialize, Serializer, de}; @@ -203,6 +203,10 @@ pub struct EndpointProfile { pub graphql_persisted_queries: HashMap, #[serde(default, skip_serializing_if = "is_zero")] pub graphql_max_body_bytes: u32, + #[serde(default, skip_serializing_if = "is_zero")] + pub json_rpc_max_body_bytes: u32, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mcp: Option, #[serde(default, skip_serializing_if = "String::is_empty")] pub path: String, #[serde(default, skip_serializing_if = "String::is_empty")] @@ -213,6 +217,14 @@ pub struct EndpointProfile { pub signing_region: String, } +#[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct McpOptionsProfile { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub strict_tool_names: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub allow_all_known_mcp_methods: Option, +} + #[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] pub struct L7RuleProfile { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -780,6 +792,8 @@ fn endpoint_to_proto(endpoint: &EndpointProfile) -> NetworkEndpoint { .map(|(name, operation)| (name.clone(), graphql_operation_to_proto(operation))) .collect(), graphql_max_body_bytes: endpoint.graphql_max_body_bytes, + json_rpc_max_body_bytes: endpoint.json_rpc_max_body_bytes, + mcp: endpoint.mcp.as_ref().map(mcp_options_to_proto), path: endpoint.path.clone(), credential_signing: endpoint.credential_signing.clone(), signing_service: endpoint.signing_service.clone(), @@ -813,6 +827,8 @@ fn endpoint_from_proto(endpoint: &NetworkEndpoint) -> EndpointProfile { .map(|(name, operation)| (name.clone(), graphql_operation_from_proto(operation))) .collect(), graphql_max_body_bytes: endpoint.graphql_max_body_bytes, + json_rpc_max_body_bytes: endpoint.json_rpc_max_body_bytes, + mcp: endpoint.mcp.map(mcp_options_from_proto), path: endpoint.path.clone(), credential_signing: endpoint.credential_signing.clone(), signing_service: endpoint.signing_service.clone(), @@ -820,6 +836,20 @@ fn endpoint_from_proto(endpoint: &NetworkEndpoint) -> EndpointProfile { } } +fn mcp_options_to_proto(options: &McpOptionsProfile) -> McpOptions { + McpOptions { + strict_tool_names: options.strict_tool_names, + allow_all_known_mcp_methods: options.allow_all_known_mcp_methods, + } +} + +fn mcp_options_from_proto(options: McpOptions) -> McpOptionsProfile { + McpOptionsProfile { + strict_tool_names: options.strict_tool_names, + allow_all_known_mcp_methods: options.allow_all_known_mcp_methods, + } +} + fn binary_to_proto(binary: &BinaryProfile) -> NetworkBinary { NetworkBinary { path: binary.path.clone(), @@ -859,6 +889,7 @@ fn allow_to_proto(allow: &L7AllowProfile) -> L7Allow { operation_type: allow.operation_type.clone(), operation_name: allow.operation_name.clone(), fields: allow.fields.clone(), + params: HashMap::new(), } } @@ -891,6 +922,7 @@ fn deny_rule_to_proto(rule: &L7DenyRuleProfile) -> L7DenyRule { operation_type: rule.operation_type.clone(), operation_name: rule.operation_name.clone(), fields: rule.fields.clone(), + params: HashMap::new(), } } @@ -1939,6 +1971,46 @@ discovery: assert!(exported.contains("api_key")); } + #[test] + fn mcp_endpoint_strict_tool_names_round_trips_through_proto_and_yaml() { + let profile = parse_profile_yaml( + r" +id: mcp-example +display_name: MCP Example +endpoints: + - host: mcp.example.com + port: 443 + path: /mcp + protocol: mcp + mcp: + strict_tool_names: false +binaries: + - /usr/bin/example-agent +", + ) + .expect("profile should parse"); + + assert_eq!( + profile.endpoints[0] + .mcp + .as_ref() + .and_then(|options| options.strict_tool_names), + Some(false) + ); + let from_proto = ProviderTypeProfile::from_proto(&profile.to_proto()); + assert_eq!( + from_proto.endpoints[0] + .mcp + .as_ref() + .and_then(|options| options.strict_tool_names), + Some(false) + ); + + let exported = profile_to_yaml(&from_proto).expect("yaml"); + assert!(exported.contains("mcp:")); + assert!(exported.contains("strict_tool_names: false")); + } + #[test] fn profile_refresh_metadata_round_trips_through_proto_and_yaml() { let profile = parse_profile_yaml( diff --git a/crates/openshell-sandbox/src/mechanistic_mapper.rs b/crates/openshell-sandbox/src/mechanistic_mapper.rs index ba7c51de99..8ee2fc37f9 100644 --- a/crates/openshell-sandbox/src/mechanistic_mapper.rs +++ b/crates/openshell-sandbox/src/mechanistic_mapper.rs @@ -355,6 +355,7 @@ fn build_l7_rules(samples: &HashMap<(String, String), u32>) -> Vec { operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::new(), }), }); } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 8b3b20bec2..cc8ff0d2e2 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -8328,6 +8328,7 @@ mod tests { operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::default(), }), }], }; @@ -8723,6 +8724,7 @@ mod tests { operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::default(), }), }], }]; @@ -8869,6 +8871,7 @@ mod tests { operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::default(), }), }], }; diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 57d1ef4bb8..7d0079f7b3 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -42,6 +42,7 @@ spiffe = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } tokio-rustls = { workspace = true } +tower-mcp-types = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } webpki-roots = { workspace = true } diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index afcd288639..efcdf07320 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -257,6 +257,7 @@ deny_request if { # --- L7 deny rule matching: REST method + path + query --- request_denied_for_endpoint(request, endpoint) if { + not jsonrpc_family_endpoint(endpoint) some deny_rule deny_rule := endpoint.deny_rules[_] deny_rule.method @@ -274,6 +275,23 @@ request_denied_for_endpoint(request, endpoint) if { command_matches(request.command, deny_rule.command) } +# --- L7 deny rule matching: JSON-RPC method --- + +request_denied_for_endpoint(request, endpoint) if { + jsonrpc_family_endpoint(endpoint) + request.method == "POST" + some deny_rule + deny_rule := endpoint.deny_rules[_] + deny_rule.method + jsonrpc_rule_matches(request, endpoint, deny_rule) +} + +request_denied_for_endpoint(request, endpoint) if { + endpoint.protocol == "json-rpc" + request.method == "POST" + jsonrpc_response_frame_present(request) +} + # --- L7 deny rule matching: GraphQL operation --- request_denied_for_endpoint(request, endpoint) if { @@ -382,10 +400,18 @@ request_deny_reason := reason if { reason := "GraphQL operation not permitted by policy" } +request_deny_reason := reason if { + input.request + jsonrpc_response_frame_present(input.request) + matched_endpoint_config.protocol == "json-rpc" + reason := "JSON-RPC response frames are not permitted from client to server" +} + request_deny_reason := reason if { input.request deny_request not graphql_request_has_operations(input.request) + not jsonrpc_response_frame_present(input.request) reason := sprintf("%s %s blocked by deny rule", [input.request.method, input.request.path]) } @@ -394,12 +420,14 @@ request_deny_reason := reason if { not deny_request not allow_request not graphql_request_has_operations(input.request) + not jsonrpc_response_frame_present(input.request) reason := sprintf("%s %s not permitted by policy", [input.request.method, input.request.path]) } # --- L7 rule matching: REST method + path --- request_allowed_for_endpoint(request, endpoint) if { + not jsonrpc_family_endpoint(endpoint) some rule rule := endpoint.rules[_] rule.allow.method @@ -417,6 +445,82 @@ request_allowed_for_endpoint(request, endpoint) if { command_matches(request.command, rule.allow.command) } +# --- L7 rule matching: JSON-RPC method --- + +request_allowed_for_endpoint(request, endpoint) if { + jsonrpc_family_endpoint(endpoint) + request.method == "POST" + some rule + rule := endpoint.rules[_] + rule.allow.method + not jsonrpc_response_frame_present(request) + jsonrpc_rule_matches(request, endpoint, rule.allow) +} + +# MCP can allow the method layer by endpoint option while still using +# tool-specific rules to narrow tools/call params.name. +request_allowed_for_endpoint(request, endpoint) if { + endpoint.protocol == "mcp" + mcp_allow_all_known_mcp_methods(endpoint) + request.method == "POST" + not jsonrpc_response_frame_present(request) + jsonrpc := object.get(request, "jsonrpc", null) + is_object(jsonrpc) + jsonrpc_no_parse_error(jsonrpc) + method := object.get(jsonrpc, "method", "") + is_string(method) + method != "" + not mcp_tool_call_narrowed_by_policy(endpoint, method) +} + +# MCP Streamable HTTP allows client-to-server JSON-RPC response frames for +# server-originated requests such as elicitation/create. Generic JSON-RPC keeps +# response frames denied because it has no MCP request-correlation semantics. +request_allowed_for_endpoint(request, endpoint) if { + endpoint.protocol == "mcp" + request.method == "POST" + jsonrpc_response_frame_present(request) + jsonrpc := object.get(request, "jsonrpc", null) + is_object(jsonrpc) + jsonrpc_no_parse_error(jsonrpc) +} + +jsonrpc_family_endpoint(endpoint) if { + endpoint.protocol == "json-rpc" +} + +jsonrpc_family_endpoint(endpoint) if { + endpoint.protocol == "mcp" +} + +mcp_allow_all_known_mcp_methods(endpoint) if { + object.get(endpoint, "mcp_allow_all_known_mcp_methods", false) +} + +mcp_tool_call_narrowed_by_policy(endpoint, method) if { + method == "tools/call" + some rule + rule := endpoint.rules[_] + params := object.get(rule.allow, "params", {}) + is_object(params) + params.name +} + +# MCP Streamable HTTP uses GET on the JSON-RPC-family endpoint as a receive +# stream for server-to-client messages. The stream itself has no +# client-to-server JSON-RPC request body to inspect; allow it once the endpoint +# path and binary matched. +request_allowed_for_endpoint(request, endpoint) if { + endpoint.protocol == "mcp" + request.method == "GET" + jsonrpc := object.get(request, "jsonrpc", null) + is_object(jsonrpc) + object.get(jsonrpc, "receive_stream", false) + jsonrpc_no_parse_error(jsonrpc) + object.get(jsonrpc, "method", null) == null + not object.get(jsonrpc, "has_response", false) +} + # --- L7 rule matching: GraphQL operation --- request_allowed_for_endpoint(request, endpoint) if { @@ -638,6 +742,85 @@ query_value_matches(value, matcher) if { glob.match(any_patterns[i], [], value) } +# JSON-RPC-family method matching. Generic JSON-RPC policies match only method. +# MCP policies may also match params.name from tool aliases. +jsonrpc_rule_matches(request, endpoint, rule) if { + jsonrpc := object.get(request, "jsonrpc", null) + is_object(jsonrpc) + method := object.get(jsonrpc, "method", "") + is_string(method) + method != "" + rule_method := object.get(rule, "method", "") + is_string(rule_method) + rule_method != "" + jsonrpc_rule_method_matches(endpoint, method, rule_method) + jsonrpc_rule_params_match_for_protocol(jsonrpc, endpoint, rule) +} + +jsonrpc_rule_method_matches(endpoint, _, rule_method) if { + endpoint.protocol == "json-rpc" + rule_method == "*" +} + +jsonrpc_rule_method_matches(endpoint, method, rule_method) if { + endpoint.protocol == "json-rpc" + rule_method != "*" + rule_method == method +} + +jsonrpc_rule_method_matches(endpoint, method, rule_method) if { + endpoint.protocol == "mcp" + glob.match(rule_method, [], method) +} + +jsonrpc_rule_params_match_for_protocol(_, endpoint, _) if { + endpoint.protocol == "json-rpc" +} + +jsonrpc_rule_params_match_for_protocol(jsonrpc, endpoint, rule) if { + endpoint.protocol == "mcp" + jsonrpc_params_match(jsonrpc, rule) +} + +jsonrpc_response_frame_present(request) if { + jsonrpc := object.get(request, "jsonrpc", null) + is_object(jsonrpc) + object.get(jsonrpc, "has_response", false) +} + +jsonrpc_no_parse_error(jsonrpc) if { + is_object(jsonrpc) + object.get(jsonrpc, "error", null) == null +} + +jsonrpc_no_parse_error(jsonrpc) if { + is_object(jsonrpc) + object.get(jsonrpc, "error", "") == "" +} + +jsonrpc_params_match(jsonrpc, rule) if { + is_object(jsonrpc) + param_rules := object.get(rule, "params", {}) + is_object(param_rules) + not jsonrpc_param_mismatch(jsonrpc, param_rules) +} + +jsonrpc_param_mismatch(jsonrpc, param_rules) if { + some key + matcher := param_rules[key] + not jsonrpc_param_key_matches(jsonrpc, key, matcher) +} + +jsonrpc_param_key_matches(jsonrpc, key, matcher) if { + is_object(jsonrpc) + params := object.get(jsonrpc, "params", {}) + is_object(params) + value := object.get(params, key, null) + value != null + is_string(value) + query_value_matches(value, matcher) +} + # SQL command matching: "*" matches any; otherwise case-insensitive. command_matches(_, "*") if true diff --git a/crates/openshell-supervisor-network/src/l7/graphql.rs b/crates/openshell-supervisor-network/src/l7/graphql.rs index 82c35720e0..12979f0b19 100644 --- a/crates/openshell-supervisor-network/src/l7/graphql.rs +++ b/crates/openshell-supervisor-network/src/l7/graphql.rs @@ -810,6 +810,7 @@ network_policies: target: req.target, query_params: req.query_params, graphql: Some(info), + jsonrpc: None, }; let tunnel_engine = engine diff --git a/crates/openshell-supervisor-network/src/l7/http.rs b/crates/openshell-supervisor-network/src/l7/http.rs new file mode 100644 index 0000000000..66269f6ba2 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/http.rs @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared HTTP/1.1 request helpers for L7 protocols carried over HTTP. + +use crate::l7::provider::{BodyLength, L7Request}; +use miette::{IntoDiagnostic, Result, miette}; +use tokio::io::{AsyncRead, AsyncReadExt}; + +const READ_BUF_SIZE: usize = 8192; + +pub async fn read_body_for_inspection( + client: &mut C, + request: &mut L7Request, + max_body_bytes: usize, +) -> Result> { + let header_end = request + .raw_header + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(request.raw_header.len(), |p| p + 4); + let overflow = request.raw_header[header_end..].to_vec(); + + match request.body_length { + BodyLength::None => Ok(Vec::new()), + BodyLength::ContentLength(len) => { + let len = usize::try_from(len) + .map_err(|_| miette!("HTTP request body length exceeds platform limit"))?; + if len > max_body_bytes { + return Err(miette!( + "HTTP request body exceeds {max_body_bytes} byte inspection limit" + )); + } + if overflow.len() > len { + return Err(miette!( + "HTTP request contains more body bytes than Content-Length" + )); + } + let remaining = len - overflow.len(); + let mut body = overflow; + if remaining > 0 { + let start = body.len(); + body.resize(len, 0); + client + .read_exact(&mut body[start..]) + .await + .into_diagnostic()?; + } + request.raw_header.truncate(header_end); + request.raw_header.extend_from_slice(&body); + Ok(body) + } + BodyLength::Chunked => { + let body = read_chunked_body_for_inspection( + client, + request, + header_end, + overflow, + max_body_bytes, + ) + .await?; + normalize_chunked_request_to_content_length(request, header_end, &body)?; + Ok(body) + } + } +} + +fn normalize_chunked_request_to_content_length( + request: &mut L7Request, + header_end: usize, + body: &[u8], +) -> Result<()> { + let header_str = std::str::from_utf8(&request.raw_header[..header_end]) + .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + let header_str = header_str + .strip_suffix("\r\n\r\n") + .ok_or_else(|| miette!("HTTP headers missing terminator"))?; + + let mut normalized = Vec::with_capacity(header_str.len() + body.len() + 32); + for (idx, line) in header_str.split("\r\n").enumerate() { + if idx > 0 { + let name = line + .split_once(':') + .map(|(name, _)| name.trim().to_ascii_lowercase()); + if matches!( + name.as_deref(), + Some("transfer-encoding" | "content-length" | "trailer") + ) { + continue; + } + } + normalized.extend_from_slice(line.as_bytes()); + normalized.extend_from_slice(b"\r\n"); + } + normalized.extend_from_slice(format!("Content-Length: {}\r\n\r\n", body.len()).as_bytes()); + normalized.extend_from_slice(body); + + request.raw_header = normalized; + request.body_length = BodyLength::ContentLength(body.len() as u64); + Ok(()) +} + +async fn read_chunked_body_for_inspection( + client: &mut C, + request: &mut L7Request, + header_end: usize, + overflow: Vec, + max_body_bytes: usize, +) -> Result> { + let mut raw = overflow; + let mut decoded = Vec::new(); + let mut pos = 0usize; + + loop { + let size_line_end = loop { + if let Some(end) = find_crlf(&raw, pos) { + break end; + } + read_more(client, &mut raw, max_body_bytes).await?; + }; + let size_line = std::str::from_utf8(&raw[pos..size_line_end]) + .into_diagnostic() + .map_err(|_| miette!("Invalid UTF-8 in HTTP chunk-size line"))?; + let size_token = size_line + .split(';') + .next() + .map(str::trim) + .unwrap_or_default(); + let chunk_size = usize::from_str_radix(size_token, 16) + .into_diagnostic() + .map_err(|_| miette!("Invalid HTTP chunk size token: {size_token:?}"))?; + pos = size_line_end + 2; + + if decoded.len().saturating_add(chunk_size) > max_body_bytes { + return Err(miette!( + "HTTP request body exceeds {max_body_bytes} byte inspection limit" + )); + } + + if chunk_size == 0 { + loop { + let trailer_end = loop { + if let Some(end) = find_crlf(&raw, pos) { + break end; + } + read_more(client, &mut raw, max_body_bytes).await?; + }; + let trailer_line = &raw[pos..trailer_end]; + pos = trailer_end + 2; + if trailer_line.is_empty() { + request.raw_header.truncate(header_end); + request.raw_header.extend_from_slice(&raw[..pos]); + return Ok(decoded); + } + } + } + + let chunk_end = pos + .checked_add(chunk_size) + .ok_or_else(|| miette!("HTTP chunk size overflow"))?; + let chunk_with_crlf_end = chunk_end + .checked_add(2) + .ok_or_else(|| miette!("HTTP chunk size overflow"))?; + while raw.len() < chunk_with_crlf_end { + read_more(client, &mut raw, max_body_bytes).await?; + } + decoded.extend_from_slice(&raw[pos..chunk_end]); + if raw.get(chunk_end..chunk_with_crlf_end) != Some(&b"\r\n"[..]) { + return Err(miette!("HTTP chunk payload missing terminating CRLF")); + } + pos = chunk_with_crlf_end; + } +} + +async fn read_more( + client: &mut C, + raw: &mut Vec, + max_body_bytes: usize, +) -> Result<()> { + if raw.len() > max_body_bytes.saturating_mul(2).max(max_body_bytes) { + return Err(miette!( + "HTTP chunked request body exceeds inspection framing limit" + )); + } + let mut buf = [0u8; READ_BUF_SIZE]; + let n = client.read(&mut buf).await.into_diagnostic()?; + if n == 0 { + return Err(miette!("HTTP chunked body ended before terminator")); + } + raw.extend_from_slice(&buf[..n]); + Ok(()) +} + +fn find_crlf(buf: &[u8], start: usize) -> Option { + buf.get(start..)? + .windows(2) + .position(|w| w == b"\r\n") + .map(|p| start + p) +} diff --git a/crates/openshell-supervisor-network/src/l7/jsonrpc.rs b/crates/openshell-supervisor-network/src/l7/jsonrpc.rs new file mode 100644 index 0000000000..7122edeee0 --- /dev/null +++ b/crates/openshell-supervisor-network/src/l7/jsonrpc.rs @@ -0,0 +1,750 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! JSON-RPC 2.0 over HTTP L7 inspection. + +use std::collections::HashMap; + +use miette::Result; +use tokio::io::{AsyncRead, AsyncWrite}; +use tower_mcp_types::protocol::{ + JSONRPC_VERSION, JsonRpcNotification, JsonRpcRequest, McpNotification, McpRequest, +}; + +use crate::l7::provider::{L7Provider, L7Request}; + +pub const DEFAULT_MAX_BODY_BYTES: usize = 64 * 1024; + +/// Selects whether the parser should treat a JSON-RPC message as generic +/// JSON-RPC 2.0 or as an MCP message with MCP method/params validation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JsonRpcInspectionMode { + JsonRpc, + Mcp, +} + +impl JsonRpcInspectionMode { + pub(crate) fn for_protocol(protocol: crate::l7::L7Protocol) -> Self { + match protocol { + crate::l7::L7Protocol::Mcp => Self::Mcp, + _ => Self::JsonRpc, + } + } +} + +/// Endpoint-specific JSON-RPC-family parser settings. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JsonRpcInspectionOptions { + pub mode: JsonRpcInspectionMode, + pub mcp_strict_tool_names: bool, +} + +impl JsonRpcInspectionOptions { + pub(crate) fn for_config(config: &crate::l7::L7EndpointConfig) -> Self { + Self { + mode: JsonRpcInspectionMode::for_protocol(config.protocol), + mcp_strict_tool_names: config.mcp_strict_tool_names, + } + } +} + +impl From for JsonRpcInspectionOptions { + fn from(mode: JsonRpcInspectionMode) -> Self { + Self { + mode, + mcp_strict_tool_names: true, + } + } +} + +/// Parsed HTTP request plus the JSON-RPC-family metadata extracted from the +/// body. The original HTTP request is still forwarded if policy allows it. +pub struct JsonRpcHttpRequest { + pub request: L7Request, + pub info: JsonRpcRequestInfo, +} + +pub(crate) async fn parse_jsonrpc_http_request( + client: &mut C, + max_body_bytes: usize, + canonicalize_options: crate::l7::path::CanonicalizeOptions, + inspection_options: JsonRpcInspectionOptions, +) -> Result> { + let provider = crate::l7::rest::RestProvider::with_options(canonicalize_options); + let Some(mut request) = provider.parse_request(client).await? else { + return Ok(None); + }; + if jsonrpc_receive_stream_request(&request) { + return Ok(Some(JsonRpcHttpRequest { + request, + info: JsonRpcRequestInfo::receive_stream(), + })); + } + let body = + crate::l7::http::read_body_for_inspection(client, &mut request, max_body_bytes).await?; + let info = parse_jsonrpc_body_with_options(&body, inspection_options); + Ok(Some(JsonRpcHttpRequest { request, info })) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JsonRpcRequestInfo { + /// Calls found in the request body. Responses and receive-stream GETs have + /// no calls but are still represented so policy can allow relay behavior. + pub calls: Vec, + pub is_batch: bool, + pub receive_stream: bool, + pub has_response: bool, + pub error: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JsonRpcCallInfo { + /// JSON-RPC method, or the MCP method name after typed MCP parsing. + pub method: String, + /// Policy-visible params for JSON-RPC-family matching. Generic JSON-RPC + /// leaves this empty because params matching is not supported. MCP exposes + /// only `params.name` for tools/call tool selection. + pub params: HashMap, + /// MCP `tools/call` tool name when known. Generic JSON-RPC leaves this + /// unset because params are not inspected. + pub tool: Option, +} + +impl JsonRpcRequestInfo { + /// MCP streamable HTTP uses an empty GET to receive server messages. It has + /// no request body to inspect, but it must still pass through MCP endpoints. + pub(crate) fn receive_stream() -> Self { + Self { + calls: Vec::new(), + is_batch: false, + receive_stream: true, + has_response: false, + error: None, + } + } +} + +pub(crate) fn jsonrpc_receive_stream_request(request: &L7Request) -> bool { + request.action.eq_ignore_ascii_case("GET") + && matches!( + request.body_length, + crate::l7::provider::BodyLength::None + | crate::l7::provider::BodyLength::ContentLength(0) + ) + && request_accepts_sse(request) +} + +fn request_accepts_sse(request: &L7Request) -> bool { + let header_end = request + .raw_header + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(request.raw_header.len(), |p| p + 4); + let header = String::from_utf8_lossy(&request.raw_header[..header_end]); + header.lines().skip(1).any(|line| { + let Some((name, value)) = line.split_once(':') else { + return false; + }; + name.trim().eq_ignore_ascii_case("accept") + && value.split(',').any(|part| { + part.split(';').next().is_some_and(|media_type| { + media_type.trim().eq_ignore_ascii_case("text/event-stream") + }) + }) + }) +} +/// Parse a JSON-RPC-family body using the endpoint's inspection mode. +pub fn parse_jsonrpc_body( + body: &[u8], + inspection_mode: JsonRpcInspectionMode, +) -> JsonRpcRequestInfo { + parse_jsonrpc_body_with_options(body, inspection_mode.into()) +} + +/// Parse a JSON-RPC-family body using the endpoint's inspection options. +pub fn parse_jsonrpc_body_with_options( + body: &[u8], + inspection_options: JsonRpcInspectionOptions, +) -> JsonRpcRequestInfo { + let Ok(value) = serde_json::from_slice::(body) else { + return JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: false, + receive_stream: false, + has_response: false, + error: Some("invalid JSON".to_string()), + }; + }; + + if let serde_json::Value::Array(items) = value { + if items.is_empty() { + return JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: true, + receive_stream: false, + has_response: false, + error: Some("empty batch".to_string()), + }; + } + let mut calls = Vec::new(); + let mut has_response = false; + for item in &items { + match parse_jsonrpc_message(item, inspection_options) { + Ok(JsonRpcMessageInfo::Call(call)) => calls.push(call), + Ok(JsonRpcMessageInfo::Response) => has_response = true, + Err(error) => { + return JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: true, + receive_stream: false, + has_response: false, + error: Some(format!("batch item invalid: {error}")), + }; + } + } + } + return JsonRpcRequestInfo { + calls, + is_batch: true, + receive_stream: false, + has_response, + error: None, + }; + } + + match parse_jsonrpc_message(&value, inspection_options) { + Ok(JsonRpcMessageInfo::Call(call)) => JsonRpcRequestInfo { + calls: vec![call], + is_batch: false, + receive_stream: false, + has_response: false, + error: None, + }, + Ok(JsonRpcMessageInfo::Response) => JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: false, + receive_stream: false, + has_response: true, + error: None, + }, + Err(error) => JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: false, + receive_stream: false, + has_response: false, + error: Some(error), + }, + } +} + +enum JsonRpcMessageInfo { + Call(JsonRpcCallInfo), + Response, +} + +// Shared framing for JSON-RPC-family messages. MCP-specific validation starts +// only after the common JSON-RPC version/method/response checks. +fn parse_jsonrpc_message( + value: &serde_json::Value, + inspection_options: JsonRpcInspectionOptions, +) -> std::result::Result { + let version = value + .get("jsonrpc") + .and_then(|v| v.as_str()) + .ok_or_else(|| "missing or non-string 'jsonrpc' field".to_string())?; + if version != JSONRPC_VERSION { + return Err(format!("unsupported JSON-RPC version '{version}'")); + } + + let has_method = value.get("method").is_some(); + let has_response_payload = jsonrpc_response_payload_present(value); + if has_method && has_response_payload { + return Err("JSON-RPC message includes both method and result/error".to_string()); + } + + if has_response_payload { + parse_jsonrpc_response(value)?; + return Ok(JsonRpcMessageInfo::Response); + } + + if has_method { + return parse_jsonrpc_call(value, inspection_options).map(JsonRpcMessageInfo::Call); + } + + Err("missing or non-string 'method' field".to_string()) +} + +fn parse_jsonrpc_call( + value: &serde_json::Value, + inspection_options: JsonRpcInspectionOptions, +) -> std::result::Result { + // MCP mode delegates method-specific validation to tower-mcp-types. The + // generic mode intentionally remains looser for non-MCP JSON-RPC servers. + if inspection_options.mode == JsonRpcInspectionMode::Mcp { + return parse_mcp_call(value, inspection_options.mcp_strict_tool_names); + } + + let method = value + .get("method") + .and_then(|m| m.as_str()) + .ok_or_else(|| "missing or non-string 'method' field".to_string())?; + Ok(JsonRpcCallInfo { + method: method.to_string(), + params: HashMap::new(), + tool: None, + }) +} + +fn jsonrpc_response_payload_present(value: &serde_json::Value) -> bool { + value.get("result").is_some() || value.get("error").is_some() +} + +fn parse_jsonrpc_response(value: &serde_json::Value) -> std::result::Result<(), String> { + let has_result = value.get("result").is_some(); + let has_error = value.get("error").is_some(); + match (has_result, has_error) { + (true, true) => return Err("JSON-RPC response includes both result and error".to_string()), + (false, false) => return Err("JSON-RPC response missing result or error".to_string()), + _ => {} + } + + let id = value + .get("id") + .ok_or_else(|| "JSON-RPC response missing id".to_string())?; + if !(id.is_string() || id.is_number() || id.is_null()) { + return Err("JSON-RPC response id must be string, number, or null".to_string()); + } + + if let Some(error) = value.get("error") + && !error.is_object() + { + return Err("JSON-RPC response error must be an object".to_string()); + } + + Ok(()) +} + +fn parse_mcp_call( + value: &serde_json::Value, + strict_tool_names: bool, +) -> std::result::Result { + if value.get("id").is_some() { + // Typed parsing validates known MCP params, but policy method profiles + // stay OpenShell-owned; see McpOptions in proto/sandbox.proto. + let request: JsonRpcRequest = serde_json::from_value(value.clone()) + .map_err(|error| format!("invalid MCP request: {error}"))?; + request + .validate() + .map_err(|error| format!("invalid MCP request: {error:?}"))?; + let mcp_request = McpRequest::from_jsonrpc(&request) + .map_err(|error| format!("invalid MCP request params: {error}"))?; + let tool = mcp_tool_name(&mcp_request); + if strict_tool_names && let Some(tool_name) = tool.as_deref() { + validate_mcp_tool_name(tool_name)?; + } + + let params = mcp_policy_params(tool.as_deref()); + return Ok(JsonRpcCallInfo { + method: mcp_request.method_name().to_string(), + params, + tool, + }); + } + + // Notifications have no id and no response expectation. Validate them as + // MCP notifications but keep extension notifications addressable. + let notification: JsonRpcNotification = serde_json::from_value(value.clone()) + .map_err(|error| format!("invalid MCP notification: {error}"))?; + if notification.jsonrpc != JSONRPC_VERSION { + return Err(format!( + "unsupported JSON-RPC version '{}'", + notification.jsonrpc + )); + } + McpNotification::from_jsonrpc(¬ification) + .map_err(|error| format!("invalid MCP notification params: {error}"))?; + + Ok(JsonRpcCallInfo { + method: notification.method, + params: HashMap::new(), + tool: None, + }) +} + +fn mcp_tool_name(request: &McpRequest) -> Option { + if let McpRequest::CallTool(params) = request { + Some(params.name.clone()) + } else { + None + } +} + +fn mcp_policy_params(tool: Option<&str>) -> HashMap { + let mut params = HashMap::new(); + if let Some(tool) = tool { + params.insert("name".to_string(), tool.to_string()); + } + params +} + +// OpenShell's default MCP hardening enforces the spec-recommended tool-name +// boundary for tools/call. See McpOptions in proto/sandbox.proto for sources. +fn validate_mcp_tool_name(name: &str) -> std::result::Result<(), String> { + if name.is_empty() + || name.len() > 128 + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.')) + { + return Err( + "MCP tool name must match ^[A-Za-z0-9_.-]{1,128}$ when strict_tool_names is enabled" + .to_string(), + ); + } + Ok(()) +} +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn parses_method_from_request_body() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + assert_eq!( + info.calls.first().map(|call| call.method.as_str()), + Some("initialize") + ); + assert_eq!(info.calls.len(), 1); + assert!(!info.is_batch); + assert!(!info.has_response); + assert!(info.error.is_none()); + } + + #[test] + fn parses_jsonrpc_response_body_without_method() { + let body = br#"{"jsonrpc":"2.0","id":1,"result":{"action":"accept","content":{}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert!(!info.is_batch); + assert!(info.has_response); + assert!(info.error.is_none()); + } + + #[test] + fn parses_jsonrpc_error_response_body_without_method() { + let body = + br#"{"jsonrpc":"2.0","id":"request-1","error":{"code":-32603,"message":"failed"}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert!(info.has_response); + assert!(info.error.is_none()); + } + + #[test] + fn ignores_params_when_extracting_method() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":{"query":"quarterly","filters":{"scope":"workspace/main"}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + assert!(info.error.is_none()); + assert_eq!( + info.calls.first().map(|call| call.method.as_str()), + Some("reports.search") + ); + let call = info.calls.first().expect("single request call"); + assert!(call.params.is_empty()); + assert!(call.tool.is_none()); + } + + #[test] + fn ignores_dotted_param_collisions_for_generic_jsonrpc() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":{"filters.scope":{"value":"literal"},"filters":{"scope.value":"nested"}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.error.is_none(), "params should be ignored: {info:?}"); + assert_eq!( + info.calls.first().map(|call| call.method.as_str()), + Some("reports.search") + ); + assert!( + info.calls + .first() + .is_some_and(|call| call.params.is_empty() && call.tool.is_none()) + ); + } + + #[test] + fn mcp_mode_validates_known_methods_and_extracts_tool() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"search_web","arguments":{"query":"openshell"}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::Mcp); + + assert!(info.error.is_none(), "expected valid MCP call: {info:?}"); + let call = info.calls.first().expect("single MCP call"); + assert_eq!(call.method, "tools/call"); + assert_eq!(call.tool.as_deref(), Some("search_web")); + assert_eq!( + call.params.get("name").map(String::as_str), + Some("search_web") + ); + assert_eq!(call.params.len(), 1); + } + + #[test] + fn mcp_mode_rejects_non_recommended_tool_names_by_default() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"read status","arguments":{}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::Mcp); + + assert!(info.calls.is_empty()); + assert!( + info.error + .as_deref() + .is_some_and(|error| error.contains("strict_tool_names")), + "expected strict tool-name error, got {info:?}" + ); + } + + #[test] + fn mcp_mode_can_disable_strict_tool_names() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"read status","arguments":{}}}"#; + let info = parse_jsonrpc_body_with_options( + body, + JsonRpcInspectionOptions { + mode: JsonRpcInspectionMode::Mcp, + mcp_strict_tool_names: false, + }, + ); + + let call = info + .calls + .first() + .expect("permissive MCP call should parse"); + assert!(info.error.is_none(), "permissive MCP call failed: {info:?}"); + assert_eq!(call.tool.as_deref(), Some("read status")); + assert_eq!( + call.params.get("name").map(String::as_str), + Some("read status") + ); + } + + #[test] + fn mcp_mode_rejects_invalid_known_method_params() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"arguments":{"query":"openshell"}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::Mcp); + + assert!(info.calls.is_empty()); + assert!( + info.error + .as_deref() + .is_some_and(|error| error.contains("invalid MCP request params")), + "expected MCP params validation error, got {info:?}" + ); + } + + #[test] + fn mcp_mode_allows_unknown_extension_methods() { + let body = + br#"{"jsonrpc":"2.0","id":1,"method":"vendor/extension","params":{"name":"custom"}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::Mcp); + + assert!( + info.error.is_none(), + "extension method should remain addressable" + ); + assert_eq!( + info.calls.first().map(|call| call.method.as_str()), + Some("vendor/extension") + ); + assert!( + info.calls + .first() + .is_some_and(|call| call.params.is_empty() && call.tool.is_none()) + ); + } + + #[test] + fn mcp_mode_ignores_tool_arguments_when_extracting_policy_params() { + let body = br#"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"read_status","arguments":{"scope.key":"literal","scope":{"key":"nested"}}}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::Mcp); + let call = info.calls.first().expect("single MCP call"); + + assert!(info.error.is_none(), "expected valid MCP call: {info:?}"); + assert_eq!(call.tool.as_deref(), Some("read_status")); + assert_eq!( + call.params.get("name").map(String::as_str), + Some("read_status") + ); + assert_eq!(call.params.len(), 1); + } + + #[test] + fn accepts_any_valid_jsonrpc_params_shape() { + let body = + br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":["ignored",{"nested":true}]}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.error.is_none()); + assert_eq!( + info.calls.first().map(|call| call.method.as_str()), + Some("reports.search") + ); + assert!( + info.calls + .first() + .is_some_and(|call| call.params.is_empty() && call.tool.is_none()) + ); + } + + #[test] + fn recognizes_streamable_http_get_receive_streams() { + let request = L7Request { + action: "GET".to_string(), + target: "/rpc".to_string(), + query_params: HashMap::new(), + raw_header: b"GET /rpc HTTP/1.1\r\nHost: jsonrpc.test\r\nAccept: application/json, text/event-stream\r\n\r\n".to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + + assert!(jsonrpc_receive_stream_request(&request)); + + let info = JsonRpcRequestInfo::receive_stream(); + assert!(info.receive_stream); + assert!(info.error.is_none()); + assert!(info.calls.is_empty()); + } + + #[test] + fn bodyless_get_without_sse_accept_is_not_receive_stream() { + let request = L7Request { + action: "GET".to_string(), + target: "/rpc".to_string(), + query_params: HashMap::new(), + raw_header: + b"GET /rpc HTTP/1.1\r\nHost: jsonrpc.test\r\nAccept: application/json\r\n\r\n" + .to_vec(), + body_length: crate::l7::provider::BodyLength::None, + }; + + assert!(!jsonrpc_receive_stream_request(&request)); + } + + #[test] + fn rejects_requests_missing_jsonrpc_version() { + let body = br#"{"id":1,"method":"reports.list"}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert_eq!( + info.error.as_deref(), + Some("missing or non-string 'jsonrpc' field") + ); + } + + #[test] + fn rejects_batch_items_missing_jsonrpc_version() { + let body = br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"id":2,"method":"reports.search","params":{"query":"quarterly"}} + ]"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert!(info.is_batch); + assert_eq!( + info.error.as_deref(), + Some("batch item invalid: missing or non-string 'jsonrpc' field") + ); + } + + #[test] + fn rejects_unsupported_jsonrpc_version() { + let body = br#"{"jsonrpc":"1.0","id":1,"method":"reports.list"}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert_eq!( + info.error.as_deref(), + Some("unsupported JSON-RPC version '1.0'") + ); + } + + #[test] + fn parses_valid_batch_without_error() { + let body = br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"method":"reports.search","params":{"query":"quarterly"}} + ]"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + assert!(info.error.is_none()); + assert!(info.is_batch); + assert!(!info.has_response); + assert_eq!(info.calls.len(), 2); + assert_eq!(info.calls[0].method, "reports.list"); + assert_eq!(info.calls[1].method, "reports.search"); + } + + #[test] + fn parses_batch_with_calls_and_responses() { + let body = br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"result":{"ok":true}} + ]"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.error.is_none()); + assert!(info.is_batch); + assert!(info.has_response); + assert_eq!(info.calls.len(), 1); + assert_eq!(info.calls[0].method, "reports.list"); + } + + #[test] + fn rejects_invalid_jsonrpc_response_body() { + let body = + br#"{"jsonrpc":"2.0","id":1,"result":{},"error":{"code":-32603,"message":"failed"}}"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert!(!info.has_response); + assert_eq!( + info.error.as_deref(), + Some("JSON-RPC response includes both result and error") + ); + } + + #[test] + fn rejects_message_with_method_and_result_or_error() { + let result_body = br#"{"jsonrpc":"2.0","id":1,"method":"initialize","result":{}}"#; + let result_info = parse_jsonrpc_body(result_body, JsonRpcInspectionMode::JsonRpc); + assert!(result_info.calls.is_empty()); + assert_eq!( + result_info.error.as_deref(), + Some("JSON-RPC message includes both method and result/error") + ); + + let error_body = br#"{"jsonrpc":"2.0","id":1,"method":"initialize","error":{"code":-32603,"message":"failed"}}"#; + let error_info = parse_jsonrpc_body(error_body, JsonRpcInspectionMode::JsonRpc); + assert!(error_info.calls.is_empty()); + assert_eq!( + error_info.error.as_deref(), + Some("JSON-RPC message includes both method and result/error") + ); + } + + #[test] + fn rejects_batch_item_with_method_and_result() { + let body = br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"method":"initialize","result":{}} + ]"#; + let info = parse_jsonrpc_body(body, JsonRpcInspectionMode::JsonRpc); + + assert!(info.calls.is_empty()); + assert!(info.is_batch); + assert_eq!( + info.error.as_deref(), + Some("batch item invalid: JSON-RPC message includes both method and result/error") + ); + } +} diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index b0f52a3ffb..d10c19b8e8 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -9,7 +9,9 @@ //! evaluated against OPA policy, and either forwarded or denied. pub mod graphql; +pub(crate) mod http; pub mod inference; +pub mod jsonrpc; pub mod path; pub mod provider; pub mod relay; @@ -25,6 +27,8 @@ pub enum L7Protocol { Websocket, Graphql, Sql, + JsonRpc, + Mcp, } impl L7Protocol { @@ -34,9 +38,15 @@ impl L7Protocol { "websocket" => Some(Self::Websocket), "graphql" => Some(Self::Graphql), "sql" => Some(Self::Sql), + "json-rpc" => Some(Self::JsonRpc), + "mcp" => Some(Self::Mcp), _ => None, } } + + pub fn is_jsonrpc_family(self) -> bool { + matches!(self, Self::JsonRpc | Self::Mcp) + } } /// TLS handling mode for proxy connections. @@ -96,6 +106,11 @@ pub struct L7EndpointConfig { pub enforcement: EnforcementMode, /// Maximum GraphQL request body bytes to buffer for inspection. pub graphql_max_body_bytes: usize, + /// Maximum JSON-RPC request body bytes to buffer for inspection. + pub json_rpc_max_body_bytes: usize, + /// MCP-only strict validation for tools/call params.name. Defaults to true + /// for MCP endpoints and is ignored by other JSON-RPC-family protocols. + pub mcp_strict_tool_names: bool, /// When true, percent-encoded `/` (`%2F`) is preserved in path segments /// rather than rejected at the parser. Needed by upstreams like GitLab /// that embed `%2F` in namespaced project paths. Defaults to false. @@ -138,6 +153,8 @@ pub struct L7RequestInfo { pub query_params: std::collections::HashMap>, /// Parsed GraphQL operation metadata for GraphQL endpoints. pub graphql: Option, + /// Parsed JSON-RPC request metadata for JSON-RPC endpoints. + pub jsonrpc: Option, } /// Parse an L7 endpoint config from a regorus Value (returned by Rego query). @@ -193,6 +210,12 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { .and_then(|v| usize::try_from(v).ok()) .filter(|v| *v > 0) .unwrap_or(graphql::DEFAULT_MAX_BODY_BYTES); + let json_rpc_max_body_bytes = get_object_u64(val, "json_rpc_max_body_bytes") + .and_then(|v| usize::try_from(v).ok()) + .filter(|v| *v > 0) + .unwrap_or(jsonrpc::DEFAULT_MAX_BODY_BYTES); + let mcp_strict_tool_names = protocol == L7Protocol::Mcp + && get_object_bool(val, "mcp_strict_tool_names").unwrap_or(true); let credential_signing = match get_object_str(val, "credential_signing").as_deref() { Some("sigv4") => CredentialSigning::SigV4, @@ -231,6 +254,8 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { tls, enforcement, graphql_max_body_bytes, + json_rpc_max_body_bytes, + mcp_strict_tool_names, allow_encoded_slash, websocket_credential_rewrite, request_body_credential_rewrite, @@ -532,6 +557,332 @@ fn validate_graphql_rule( validate_graphql_fields(errors, warnings, loc, rule.get("fields")); } +// Validate a matcher map when it exists. Null is treated like omission because +// policy loading normalizes absent optional maps the same way. +fn validate_matcher_map( + errors: &mut Vec, + warnings: &mut Vec, + loc: &str, + value: Option<&serde_json::Value>, +) { + let Some(value) = value.filter(|v| !v.is_null()) else { + return; + }; + let Some(obj) = value.as_object() else { + errors.push(format!("{loc}: expected map of matchers")); + return; + }; + + for (key, matcher) in obj { + validate_matcher_value(errors, warnings, &format!("{loc}.{key}"), matcher); + } +} + +// Validate one matcher leaf. Objects must use the explicit matcher keys. +fn validate_matcher_value( + errors: &mut Vec, + warnings: &mut Vec, + loc: &str, + matcher: &serde_json::Value, +) { + if let Some(glob_str) = matcher.as_str() { + if let Some(warning) = check_glob_syntax(glob_str) { + warnings.push(format!("{loc}: {warning}")); + } + return; + } + + let Some(matcher_obj) = matcher.as_object() else { + errors.push(format!("{loc}: expected string glob or matcher object")); + return; + }; + + let has_any = matcher_obj.get("any").is_some(); + let has_glob = matcher_obj.get("glob").is_some(); + if !has_any && !has_glob { + errors.push(format!( + "{loc}: unknown matcher keys; only `glob` or `any` are supported" + )); + return; + } + + let has_unknown = matcher_obj.keys().any(|k| k != "any" && k != "glob"); + if has_unknown { + errors.push(format!( + "{loc}: unknown matcher keys; only `glob` or `any` are supported" + )); + return; + } + + if has_glob && has_any { + errors.push(format!( + "{loc}: matcher cannot specify both `glob` and `any`" + )); + return; + } + + if has_glob { + match matcher_obj.get("glob").and_then(|v| v.as_str()) { + None => errors.push(format!("{loc}.glob: expected glob string")), + Some(glob_str) => { + if let Some(warning) = check_glob_syntax(glob_str) { + warnings.push(format!("{loc}.glob: {warning}")); + } + } + } + return; + } + + let Some(any) = matcher_obj.get("any").and_then(|v| v.as_array()) else { + errors.push(format!("{loc}.any: expected array of glob strings")); + return; + }; + if any.is_empty() { + errors.push(format!("{loc}.any: list must not be empty")); + return; + } + if any.iter().any(|v| v.as_str().is_none()) { + errors.push(format!("{loc}.any: all values must be strings")); + } + for item in any.iter().filter_map(|v| v.as_str()) { + if let Some(warning) = check_glob_syntax(item) { + warnings.push(format!("{loc}.any: {warning}")); + } + } +} + +// Validate the shared JSON-RPC-family rule surface. Generic JSON-RPC requires +// an explicit method but does not support authored params matchers yet. MCP +// keeps method optional while the endpoint-level method profile is enabled; +// disabling that profile makes method explicit on every authored MCP rule. MCP +// does not expose tool-argument matching. +fn validate_jsonrpc_rule_fields( + errors: &mut Vec, + warnings: &mut Vec, + loc: &str, + rule: &serde_json::Value, + protocol: &str, + mcp_strict_tool_names: bool, + mcp_allow_all_known_mcp_methods: bool, +) { + let method = rule.get("method").and_then(|v| v.as_str()).unwrap_or(""); + let has_params = rule.get("params").is_some_and(|v| !v.is_null()); + let has_tool = rule.get("tool").is_some_and(|v| !v.is_null()); + let has_tool_selector = mcp_rule_has_tool_selector(rule); + + if protocol == "json-rpc" { + if method.is_empty() { + errors.push(format!("{loc}.method: required for {protocol} L7 rules")); + } else if method != "*" && glob_uses_wildcard(method) { + errors.push(format!( + "{loc}.method: generic JSON-RPC method rules do not support glob or wildcard matchers; use \"*\" to allow all methods or an exact method name" + )); + } + if has_params { + errors.push(format!( + "{loc}: JSON-RPC rules do not support params matchers yet" + )); + } + if has_tool { + errors.push(format!( + "{loc}.tool: MCP tool matching is only valid for protocol mcp" + )); + } + if json_rule_has_non_empty_path_or_query(rule) { + errors.push(format!( + "{loc}: {protocol} L7 rules must use method, not path/query" + )); + } + return; + } + + if protocol == "mcp" { + validate_mcp_method_field(errors, warnings, loc, method); + if method.is_empty() && !mcp_allow_all_known_mcp_methods { + errors.push(format!( + "{loc}.method: required when mcp.allow_all_known_mcp_methods is false" + )); + } else if has_tool_selector && !method.is_empty() && method != "tools/call" { + errors.push(format!( + "{loc}.method: must be tools/call when an MCP rule uses tool or params.name, got '{method}'" + )); + } + validate_mcp_tool_field(errors, warnings, loc, rule, mcp_strict_tool_names); + validate_mcp_params_field(errors, warnings, loc, rule, has_tool, mcp_strict_tool_names); + if json_rule_has_non_empty_path_or_query(rule) { + errors.push(format!( + "{loc}: {protocol} L7 rules must use method/tool, not path/query" + )); + } + return; + } + + if has_tool { + errors.push(format!( + "{loc}.tool: MCP tool matching is only valid for protocol mcp" + )); + } + + if has_params { + errors.push(format!( + "{loc}.params: params matching is only valid for protocol mcp" + )); + } +} + +fn validate_mcp_method_field( + errors: &mut Vec, + warnings: &mut Vec, + loc: &str, + method: &str, +) { + if method.is_empty() { + return; + } + if let Some(warning) = check_glob_syntax(method) { + warnings.push(format!("{loc}.method: {warning}")); + } + if glob_uses_wildcard(method) && !method.starts_with("tools/") { + errors.push(format!( + "{loc}.method: MCP method globs are only valid for the tools/ method family; omit method to use the endpoint method profile" + )); + } +} + +fn method_matcher_matches_tools_call(method: &str) -> bool { + method == "tools/call" + || method == "*" + || glob::Pattern::new(method).is_ok_and(|pattern| pattern.matches("tools/call")) +} + +fn mcp_rule_has_tool_selector(rule: &serde_json::Value) -> bool { + rule.get("tool").is_some_and(|v| !v.is_null()) + || rule + .get("params") + .and_then(serde_json::Value::as_object) + .and_then(|params| params.get("name")) + .is_some_and(|v| !v.is_null()) +} + +fn mcp_endpoint_has_tool_allow_selectors(ep: &serde_json::Value) -> bool { + ep.get("rules") + .and_then(serde_json::Value::as_array) + .is_some_and(|rules| { + rules.iter().any(|rule| { + let allow = rule.get("allow").unwrap_or(rule); + mcp_rule_has_tool_selector(allow) + }) + }) +} + +fn validate_mcp_tool_field( + errors: &mut Vec, + warnings: &mut Vec, + loc: &str, + rule: &serde_json::Value, + mcp_strict_tool_names: bool, +) { + let Some(tool) = rule.get("tool").filter(|v| !v.is_null()) else { + return; + }; + validate_matcher_value(errors, warnings, &format!("{loc}.tool"), tool); + validate_mcp_tool_name_wildcard_policy( + errors, + &format!("{loc}.tool"), + tool, + mcp_strict_tool_names, + ); +} + +fn validate_mcp_params_field( + errors: &mut Vec, + warnings: &mut Vec, + loc: &str, + rule: &serde_json::Value, + has_tool: bool, + mcp_strict_tool_names: bool, +) { + let Some(params) = rule.get("params").filter(|v| !v.is_null()) else { + return; + }; + let Some(params_obj) = params.as_object() else { + errors.push(format!("{loc}.params: expected map of matchers")); + return; + }; + + if has_tool && params_obj.contains_key("name") { + errors.push(format!( + "{loc}: MCP rules must use either tool or params.name, not both" + )); + } + + for key in params_obj.keys() { + if key != "name" { + errors.push(format!( + "{loc}.params.{key}: MCP tool argument matching is not supported yet" + )); + } + } + + if let Some(name_matcher) = params_obj.get("name") { + validate_matcher_value( + errors, + warnings, + &format!("{loc}.params.name"), + name_matcher, + ); + validate_mcp_tool_name_wildcard_policy( + errors, + &format!("{loc}.params.name"), + name_matcher, + mcp_strict_tool_names, + ); + } +} + +fn validate_mcp_tool_name_wildcard_policy( + errors: &mut Vec, + loc: &str, + matcher: &serde_json::Value, + mcp_strict_tool_names: bool, +) { + if !mcp_strict_tool_names && matcher_uses_glob_wildcard(matcher) { + errors.push(format!( + "{loc}: wildcard tool-name matchers require mcp.strict_tool_names to remain enabled" + )); + } +} + +fn matcher_uses_glob_wildcard(matcher: &serde_json::Value) -> bool { + if let Some(glob) = matcher.as_str() { + return glob_uses_wildcard(glob); + } + + let Some(obj) = matcher.as_object() else { + return false; + }; + if obj + .get("glob") + .and_then(serde_json::Value::as_str) + .is_some_and(glob_uses_wildcard) + { + return true; + } + obj.get("any") + .and_then(serde_json::Value::as_array) + .is_some_and(|values| { + values + .iter() + .filter_map(serde_json::Value::as_str) + .any(glob_uses_wildcard) + }) +} + +fn glob_uses_wildcard(glob: &str) -> bool { + glob.bytes() + .any(|b| matches!(b, b'*' | b'?' | b'[' | b']' | b'{' | b'}')) +} + fn json_rule_has_graphql_fields(rule: &serde_json::Value) -> bool { rule.get("operation_type") .and_then(|v| v.as_str()) @@ -547,6 +898,16 @@ fn json_rule_has_transport_fields(rule: &serde_json::Value) -> bool { rule.get("method").is_some() || rule.get("path").is_some() || rule.get("query").is_some() } +fn json_rule_has_non_empty_path_or_query(rule: &serde_json::Value) -> bool { + rule.get("path") + .and_then(|v| v.as_str()) + .is_some_and(|v| !v.is_empty()) + || rule + .get("query") + .and_then(|v| v.as_object()) + .is_some_and(|v| !v.is_empty()) +} + fn json_endpoint_has_graphql_policy(ep: &serde_json::Value) -> bool { ep.get("graphql_persisted_queries") .and_then(|v| v.as_object()) @@ -593,6 +954,8 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< for (i, ep) in endpoints.iter().enumerate() { let protocol = ep.get("protocol").and_then(|v| v.as_str()).unwrap_or(""); + let l7_protocol = L7Protocol::parse(protocol); + let jsonrpc_family = l7_protocol.is_some_and(L7Protocol::is_jsonrpc_family); let tls = ep.get("tls").and_then(|v| v.as_str()).unwrap_or(""); let enforcement = ep.get("enforcement").and_then(|v| v.as_str()).unwrap_or(""); let access = ep.get("access").and_then(|v| v.as_str()).unwrap_or(""); @@ -618,6 +981,19 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< ); let loc = format!("{name}.endpoints[{i}]"); + if protocol == "mcp" { + if host.trim().is_empty() { + errors.push(format!( + "{loc}: protocol mcp requires host; protocol alone is not a wildcard endpoint" + )); + } + if !ports.iter().any(|port| *port > 0) { + errors.push(format!( + "{loc}: protocol mcp requires port or ports; protocol alone is not a wildcard endpoint" + )); + } + } + if !endpoint_path.is_empty() { if !endpoint_path.starts_with('/') && endpoint_path != "**" { errors.push(format!( @@ -651,16 +1027,34 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< errors.push(format!("{loc}: rules and access are mutually exclusive")); } + if jsonrpc_family && !access.is_empty() { + if protocol == "mcp" { + errors.push(format!( + "{loc}: protocol {protocol} does not support access presets; use rules/deny_rules or set mcp.allow_all_known_mcp_methods: true for an allow-all MCP policy" + )); + } else { + errors.push(format!( + "{loc}: protocol {protocol} does not support access presets; use explicit rules with allow.method such as \"*\"" + )); + } + } + + if protocol == "json-rpc" && !has_rules { + errors.push(format!( + "{loc}: protocol {protocol} requires explicit rules with allow.method" + )); + } + // protocol requires rules or access - if !protocol.is_empty() && !has_rules && access.is_empty() { + if !protocol.is_empty() && protocol != "mcp" && !has_rules && access.is_empty() { errors.push(format!( "{loc}: protocol requires rules or access to define allowed traffic" )); } - if !protocol.is_empty() && L7Protocol::parse(protocol).is_none() { + if !protocol.is_empty() && l7_protocol.is_none() { errors.push(format!( - "{loc}: unknown protocol '{protocol}' (expected rest, websocket, graphql, or sql)" + "{loc}: unknown protocol '{protocol}' (expected rest, websocket, graphql, sql, json-rpc, or mcp)" )); } @@ -686,6 +1080,18 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< } } + if ep.get("json_rpc_max_body_bytes").is_some() { + let valid_max = ep + .get("json_rpc_max_body_bytes") + .and_then(serde_json::Value::as_u64) + .is_some_and(|v| v > 0); + if !valid_max { + errors.push(format!( + "{loc}: json_rpc_max_body_bytes must be a positive integer" + )); + } + } + if protocol != "graphql" && protocol != "websocket" && (ep.get("persisted_queries").is_some() @@ -697,6 +1103,62 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< )); } + if !jsonrpc_family && ep.get("json_rpc_max_body_bytes").is_some() { + warnings.push(format!( + "{loc}: JSON-RPC-specific endpoint fields are ignored unless protocol is json-rpc or mcp" + )); + } + let has_mcp_strict_tool_names = ep.get("mcp_strict_tool_names").is_some(); + let has_mcp_allow_all_known_mcp_methods = + ep.get("mcp_allow_all_known_mcp_methods").is_some(); + if has_mcp_strict_tool_names { + if ep + .get("mcp_strict_tool_names") + .and_then(serde_json::Value::as_bool) + .is_none() + { + errors.push(format!("{loc}: mcp.strict_tool_names must be boolean")); + } + if protocol != "mcp" { + errors.push(format!( + "{loc}: mcp.strict_tool_names is only valid for protocol mcp" + )); + } + } + if has_mcp_allow_all_known_mcp_methods { + if ep + .get("mcp_allow_all_known_mcp_methods") + .and_then(serde_json::Value::as_bool) + .is_none() + { + errors.push(format!( + "{loc}: mcp.allow_all_known_mcp_methods must be boolean" + )); + } + if protocol != "mcp" { + errors.push(format!( + "{loc}: mcp.allow_all_known_mcp_methods is only valid for protocol mcp" + )); + } + } + let mcp_strict_tool_names = ep + .get("mcp_strict_tool_names") + .and_then(serde_json::Value::as_bool) + .unwrap_or(true); + let mcp_allow_all_known_mcp_methods = ep + .get("mcp_allow_all_known_mcp_methods") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if protocol == "mcp" + && !has_rules + && access.is_empty() + && !mcp_allow_all_known_mcp_methods + { + errors.push(format!( + "{loc}: protocol mcp requires rules when mcp.allow_all_known_mcp_methods is false" + )); + } + if ep .get("websocket_credential_rewrite") .and_then(serde_json::Value::as_bool) @@ -783,16 +1245,31 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< } // deny_rules require some allow base (access or rules) - if !has_rules && access.is_empty() { + if protocol != "mcp" && !has_rules && access.is_empty() { errors.push(format!( "{loc}: deny_rules require rules or access to define the base allow set" )); } + let has_mcp_tool_allow_selectors = + protocol == "mcp" && mcp_endpoint_has_tool_allow_selectors(ep); + if let Some(deny_rules) = ep.get("deny_rules").and_then(|v| v.as_array()) { for (deny_idx, deny_rule) in deny_rules.iter().enumerate() { let deny_loc = format!("{loc}.deny_rules[{deny_idx}]"); + if has_mcp_tool_allow_selectors + && deny_rule + .get("method") + .and_then(serde_json::Value::as_str) + .is_some_and(method_matcher_matches_tools_call) + && !mcp_rule_has_tool_selector(deny_rule) + { + errors.push(format!( + "{deny_loc}: method matcher denies every tool call and conflicts with MCP tool allow rules; add tool or params.name to deny specific tools, or remove the tool allow rules" + )); + } + // Validate method if let Some(method) = deny_rule.get("method").and_then(|m| m.as_str()) && !method.is_empty() @@ -816,111 +1293,32 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< // Validate query matchers — mirrors allow-side validation exactly if let Some(query) = deny_rule.get("query").filter(|v| !v.is_null()) { - let Some(query_obj) = query.as_object() else { - errors.push(format!( - "{deny_loc}.query: expected map of query matchers" - )); - continue; - }; - - for (param, matcher) in query_obj { - if let Some(glob_str) = matcher.as_str() { - if let Some(warning) = check_glob_syntax(glob_str) { - warnings - .push(format!("{deny_loc}.query.{param}: {warning}")); - } - continue; - } - - let Some(matcher_obj) = matcher.as_object() else { - errors.push(format!( - "{deny_loc}.query.{param}: expected string glob or object with `any`" - )); - continue; - }; - - let has_any = matcher_obj.get("any").is_some(); - let has_glob = matcher_obj.get("glob").is_some(); - let has_unknown = - matcher_obj.keys().any(|k| k != "any" && k != "glob"); - if has_unknown { - errors.push(format!( - "{deny_loc}.query.{param}: unknown matcher keys; only `glob` or `any` are supported" - )); - continue; - } - - if has_glob && has_any { - errors.push(format!( - "{deny_loc}.query.{param}: matcher cannot specify both `glob` and `any`" - )); - continue; - } + validate_matcher_map( + &mut errors, + &mut warnings, + &format!("{deny_loc}.query"), + Some(query), + ); + } - if !has_glob && !has_any { - errors.push(format!( - "{deny_loc}.query.{param}: object matcher requires `glob` string or non-empty `any` list" - )); - continue; - } + validate_jsonrpc_rule_fields( + &mut errors, + &mut warnings, + &deny_loc, + deny_rule, + protocol, + mcp_strict_tool_names, + mcp_allow_all_known_mcp_methods, + ); - if has_glob { - match matcher_obj.get("glob").and_then(|v| v.as_str()) { - None => { - errors.push(format!( - "{deny_loc}.query.{param}.glob: expected glob string" - )); - } - Some(g) => { - if let Some(warning) = check_glob_syntax(g) { - warnings.push(format!( - "{deny_loc}.query.{param}.glob: {warning}" - )); - } - } - } - continue; - } - - let any = matcher_obj.get("any").and_then(|v| v.as_array()); - let Some(any) = any else { - errors.push(format!( - "{deny_loc}.query.{param}.any: expected array of glob strings" - )); - continue; - }; - - if any.is_empty() { - errors.push(format!( - "{deny_loc}.query.{param}.any: list must not be empty" - )); - continue; - } - - if any.iter().any(|v| v.as_str().is_none()) { - errors.push(format!( - "{deny_loc}.query.{param}.any: all values must be strings" - )); - } - - for item in any.iter().filter_map(|v| v.as_str()) { - if let Some(warning) = check_glob_syntax(item) { - warnings.push(format!( - "{deny_loc}.query.{param}.any: {warning}" - )); - } - } - } - } - - // SQL command validation - if let Some(command) = deny_rule.get("command").and_then(|c| c.as_str()) - && !command.is_empty() - && protocol == "rest" - { - warnings - .push(format!("{deny_loc}: command is for SQL protocol, not REST")); - } + // SQL command validation + if let Some(command) = deny_rule.get("command").and_then(|c| c.as_str()) + && !command.is_empty() + && protocol == "rest" + { + warnings + .push(format!("{deny_loc}: command is for SQL protocol, not REST")); + } let deny_has_graphql = json_rule_has_graphql_fields(deny_rule); if protocol == "websocket" @@ -986,109 +1384,42 @@ pub fn validate_l7_policies(data_json: &serde_json::Value) -> (Vec, Vec< continue; }; - let Some(query_obj) = query.as_object() else { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query: expected map of query matchers" - )); - continue; - }; - - for (param, matcher) in query_obj { - if let Some(glob_str) = matcher.as_str() { - if let Some(warning) = check_glob_syntax(glob_str) { - warnings.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}: {warning}" - )); - } - continue; - } - - let Some(matcher_obj) = matcher.as_object() else { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}: expected string glob or object with `any`" - )); - continue; - }; - - let has_any = matcher_obj.get("any").is_some(); - let has_glob = matcher_obj.get("glob").is_some(); - let has_unknown = matcher_obj.keys().any(|k| k != "any" && k != "glob"); - if has_unknown { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}: unknown matcher keys; only `glob` or `any` are supported" - )); - continue; - } - - if has_glob && has_any { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}: matcher cannot specify both `glob` and `any`" - )); - continue; - } - - if !has_glob && !has_any { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}: object matcher requires `glob` string or non-empty `any` list" - )); - continue; - } - - if has_glob { - match matcher_obj.get("glob").and_then(|v| v.as_str()) { - None => { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}.glob: expected glob string" - )); - } - Some(g) => { - if let Some(warning) = check_glob_syntax(g) { - warnings.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}.glob: {warning}" - )); - } - } - } - continue; - } - - let any = matcher_obj.get("any").and_then(|v| v.as_array()); - let Some(any) = any else { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}.any: expected array of glob strings" - )); - continue; - }; - - if any.is_empty() { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}.any: list must not be empty" - )); - continue; - } - - if any.iter().any(|v| v.as_str().is_none()) { - errors.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}.any: all values must be strings" - )); - } - - for item in any.iter().filter_map(|v| v.as_str()) { - if let Some(warning) = check_glob_syntax(item) { - warnings.push(format!( - "{loc}.rules[{rule_idx}].allow.query.{param}.any: {warning}" - )); - } - } - } + validate_matcher_map( + &mut errors, + &mut warnings, + &format!("{loc}.rules[{rule_idx}].allow.query"), + Some(query), + ); } } } + let has_mcp_tool_allow_selectors = + protocol == "mcp" && mcp_endpoint_has_tool_allow_selectors(ep); if has_rules && let Some(rules) = ep.get("rules").and_then(|v| v.as_array()) { for (rule_idx, rule) in rules.iter().enumerate() { let allow = rule.get("allow").unwrap_or(rule); let rule_loc = format!("{loc}.rules[{rule_idx}].allow"); + if has_mcp_tool_allow_selectors + && allow + .get("method") + .and_then(serde_json::Value::as_str) + .is_some_and(method_matcher_matches_tools_call) + && !mcp_rule_has_tool_selector(allow) + { + errors.push(format!( + "{rule_loc}: method matcher allows every tool call and conflicts with MCP tool allow rules; add tool or params.name to narrow tools/call, or remove the tool allow rules" + )); + } + validate_jsonrpc_rule_fields( + &mut errors, + &mut warnings, + &rule_loc, + allow, + protocol, + mcp_strict_tool_names, + mcp_allow_all_known_mcp_methods, + ); let allow_has_graphql = json_rule_has_graphql_fields(allow); if websocket_has_graphql_policy && allow @@ -1140,29 +1471,46 @@ pub fn expand_access_presets(data: &mut serde_json::Value) { }; for ep in endpoints.iter_mut() { + let protocol = ep + .get("protocol") + .and_then(|v| v.as_str()) + .unwrap_or("rest"); + let has_rules = ep + .get("rules") + .and_then(|v| v.as_array()) + .is_some_and(|a| !a.is_empty()); let access = ep .get("access") .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); + let mcp_allow_all_known_mcp_methods = ep + .get("mcp_allow_all_known_mcp_methods") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + + if protocol == "mcp" + && access.is_empty() + && !has_rules + && mcp_allow_all_known_mcp_methods + { + ep.as_object_mut().unwrap().insert( + "rules".to_string(), + serde_json::Value::Array(vec![jsonrpc_rule_json("*")]), + ); + continue; + } + if access.is_empty() { continue; } // Don't expand if rules already exist (validation will catch this) - if ep - .get("rules") - .and_then(|v| v.as_array()) - .is_some_and(|a| !a.is_empty()) - { + if has_rules { continue; } - let protocol = ep - .get("protocol") - .and_then(|v| v.as_str()) - .unwrap_or("rest"); let rules = if protocol == "graphql" { match access.as_str() { "read-only" => vec![graphql_rule_json("query")], @@ -1213,6 +1561,14 @@ fn rule_json(method: &str, path: &str) -> serde_json::Value { }) } +fn jsonrpc_rule_json(method: &str) -> serde_json::Value { + serde_json::json!({ + "allow": { + "method": method + } + }) +} + fn valid_methods_for_protocol(protocol: &str) -> &'static [&'static str] { match protocol { "websocket" => &["GET", "WEBSOCKET_TEXT", "*"], @@ -1341,6 +1697,26 @@ mod tests { assert!(config.allow_encoded_slash); } + #[test] + fn parse_l7_config_mcp_strict_tool_names_defaults_true() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "mcp", "host": "mcp.example.com", "port": 443}"#, + ) + .unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert!(config.mcp_strict_tool_names); + } + + #[test] + fn parse_l7_config_mcp_strict_tool_names_can_disable() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "mcp", "host": "mcp.example.com", "port": 443, "mcp_strict_tool_names": false}"#, + ) + .unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert!(!config.mcp_strict_tool_names); + } + #[test] fn parse_l7_config_websocket_credential_rewrite_defaults_false() { let val = regorus::Value::from_json_str( @@ -1382,136 +1758,691 @@ mod tests { } #[test] - fn parse_l7_config_websocket_graphql_policy_defaults_false() { - let val = regorus::Value::from_json_str( - r#"{"protocol": "websocket", "host": "gateway.example.com", "port": 443, "rules": [{"allow": {"method": "GET", "path": "/graphql"}}, {"allow": {"method": "WEBSOCKET_TEXT", "path": "/graphql"}}]}"#, - ) - .unwrap(); - let config = parse_l7_config(&val).unwrap(); - assert!(!config.websocket_graphql_policy); + fn parse_l7_config_websocket_graphql_policy_defaults_false() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "websocket", "host": "gateway.example.com", "port": 443, "rules": [{"allow": {"method": "GET", "path": "/graphql"}}, {"allow": {"method": "WEBSOCKET_TEXT", "path": "/graphql"}}]}"#, + ) + .unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert!(!config.websocket_graphql_policy); + } + + #[test] + fn parse_l7_config_websocket_graphql_policy_detects_operation_rules() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "websocket", "host": "gateway.example.com", "port": 443, "rules": [{"allow": {"method": "GET", "path": "/graphql"}}, {"allow": {"operation_type": "subscription", "fields": ["messageAdded"]}}]}"#, + ) + .unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert!(config.websocket_graphql_policy); + } + + #[test] + fn validate_websocket_credential_rewrite_warns_unless_rest_or_websocket() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "websocket_credential_rewrite": true + }], + "binaries": [] + } + } + }); + let (_errors, warnings) = validate_l7_policies(&data); + assert!( + warnings + .iter() + .any(|w| w.contains("websocket_credential_rewrite is ignored")), + "expected websocket_credential_rewrite warning: {warnings:?}" + ); + } + + #[test] + fn validate_request_body_credential_rewrite_warns_unless_rest() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "protocol": "websocket", + "request_body_credential_rewrite": true + }], + "binaries": [] + } + } + }); + let (_errors, warnings) = validate_l7_policies(&data); + assert!( + warnings + .iter() + .any(|w| w.contains("request_body_credential_rewrite is ignored")), + "expected request_body_credential_rewrite warning: {warnings:?}" + ); + } + + #[test] + fn expand_websocket_read_write_access_includes_text_messages() { + let mut data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "protocol": "websocket", + "access": "read-write" + }], + "binaries": [] + } + } + }); + + expand_access_presets(&mut data); + let rules = data["network_policies"]["test"]["endpoints"][0]["rules"] + .as_array() + .unwrap(); + let methods: Vec<&str> = rules + .iter() + .map(|r| r["allow"]["method"].as_str().unwrap()) + .collect(); + assert!(methods.contains(&"GET")); + assert!(methods.contains(&"WEBSOCKET_TEXT")); + } + + #[test] + fn validate_websocket_accepts_graphql_operation_rules() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "protocol": "websocket", + "rules": [ + {"allow": {"method": "GET", "path": "/graphql"}}, + {"allow": {"operation_type": "subscription", "fields": ["messageAdded"]}} + ] + }], + "binaries": [] + } + } + }); + let (errors, warnings) = validate_l7_policies(&data); + assert!(errors.is_empty(), "expected no errors: {errors:?}"); + assert!(warnings.is_empty(), "expected no warnings: {warnings:?}"); + } + + #[test] + fn validate_websocket_graphql_rule_requires_operation_type() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "protocol": "websocket", + "rules": [ + {"allow": {"method": "GET", "path": "/graphql"}}, + {"allow": {"fields": ["messageAdded"]}} + ] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| e.contains("operation_type")), + "expected missing operation_type error: {errors:?}" + ); + } + + #[test] + fn validate_websocket_graphql_rule_rejects_mixed_transport_fields() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "protocol": "websocket", + "rules": [ + {"allow": {"method": "GET", "path": "/graphql"}}, + {"allow": {"method": "WEBSOCKET_TEXT", "path": "/graphql", "operation_type": "subscription"}} + ] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| e.contains("must not combine")), + "expected mixed-field error: {errors:?}" + ); + } + + #[test] + fn validate_websocket_graphql_policy_rejects_raw_text_message_rule() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "gateway.example.com", + "port": 443, + "protocol": "websocket", + "rules": [ + {"allow": {"method": "GET", "path": "/graphql"}}, + {"allow": {"method": "WEBSOCKET_TEXT", "path": "/graphql"}}, + {"allow": {"operation_type": "query"}} + ] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors + .iter() + .any(|e| e.contains("instead of WEBSOCKET_TEXT")), + "expected raw WEBSOCKET_TEXT rejection: {errors:?}" + ); + } + + #[test] + fn validate_rules_and_access_mutual_exclusion() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "api.example.com", + "port": 443, + "protocol": "rest", + "access": "read-only", + "rules": [{"allow": {"method": "GET", "path": "**"}}] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!(errors.iter().any(|e| e.contains("mutually exclusive"))); + } + + #[test] + fn validate_jsonrpc_rejects_access_presets() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "jsonrpc.example.com", + "port": 443, + "path": "/rpc", + "protocol": "json-rpc", + "access": "full" + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| { + e.contains("json-rpc") + && e.contains("does not support access presets") + && e.contains("method") + }), + "JSON-RPC access presets should be rejected: {errors:?}" + ); + } + + #[test] + fn validate_jsonrpc_requires_method_rules() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "jsonrpc.example.com", + "port": 443, + "path": "/rpc", + "protocol": "json-rpc", + "rules": [{ + "allow": { + "path": "/rpc" + } + }] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors + .iter() + .any(|e| { e.contains("rules[0].allow.method") && e.contains("required") }), + "JSON-RPC allow rules without method should be rejected: {errors:?}" + ); + assert!( + errors + .iter() + .any(|e| { e.contains("must use method, not path/query") }), + "JSON-RPC allow rules with path/query should be rejected: {errors:?}" + ); + } + + #[test] + fn validate_mcp_tool_selectors_use_endpoint_method_profile_and_reject_arguments() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "mcp.example.com", + "port": 443, + "path": "/mcp", + "protocol": "mcp", + "mcp_allow_all_known_mcp_methods": true, + "rules": [{ + "allow": { + "tool": { "any": ["read_status", "submit_*"] } + } + }, { + "allow": { + "method": "tools/call", + "tool": "submit_report", + "params": { + "arguments": { + "scope": "workspace/main" + } + } + } + }, { + "allow": { + "method": "initialize" + } + }, { + "allow": { + "method": "tools/call", + "tool": "list_reports" + } + }], + "deny_rules": [{ + "tool": "delete_*" + }] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + !errors + .iter() + .any(|e| e.contains("rules[0].allow.method") && e.contains("required")), + "MCP tool rules can omit method while allow_all_known_mcp_methods is enabled: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("rules[1].allow.params.arguments") + && e.contains("argument matching is not supported") + }), + "MCP argument params should be rejected: {errors:?}" + ); + assert!( + !errors.iter().any(|e| e.contains("rules[2].allow.method")), + "MCP method-only rules should not require a tool selector: {errors:?}" + ); + assert!( + !errors.iter().any(|e| e.contains("rules[3].allow.method")), + "MCP tool rules with method: tools/call should validate: {errors:?}" + ); + assert!( + !errors + .iter() + .any(|e| e.contains("deny_rules[0].method") && e.contains("tools/call")), + "MCP deny tool rules can omit method while allow_all_known_mcp_methods is enabled: {errors:?}" + ); + } + + #[test] + fn validate_mcp_tool_selectors_require_method_by_default() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "mcp.example.com", + "port": 443, + "path": "/mcp", + "protocol": "mcp", + "rules": [{ + "allow": { + "tool": "read_status" + } + }], + "deny_rules": [{ + "tool": "delete_*" + }] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| { + e.contains("rules[0].allow.method") + && e.contains("mcp.allow_all_known_mcp_methods is false") + }), + "MCP allow tool rules should require method when method profile is disabled: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("deny_rules[0].method") + && e.contains("mcp.allow_all_known_mcp_methods is false") + }), + "MCP deny tool rules should require method when method profile is disabled: {errors:?}" + ); + } + + #[test] + fn validate_mcp_method_globs_are_tools_family_only() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "mcp.example.com", + "port": 443, + "path": "/mcp", + "protocol": "mcp", + "rules": [{ + "allow": { + "method": "vendor/extension" + } + }, { + "allow": { + "method": "vendor/*" + } + }, { + "allow": { + "method": "*" + } + }, { + "allow": { + "method": "tools/*" + } + }] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + !errors.iter().any(|e| e.contains("rules[0].allow.method")), + "literal extension methods should stay addressable: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("rules[1].allow.method") + && e.contains("only valid for the tools/ method family") + }), + "non-tools method globs should be rejected: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("rules[2].allow.method") + && e.contains("only valid for the tools/ method family") + }), + "authored method: * should be rejected for MCP: {errors:?}" + ); + assert!( + !errors.iter().any(|e| e.contains("rules[3].allow.method")), + "tools-family method globs should validate: {errors:?}" + ); } #[test] - fn parse_l7_config_websocket_graphql_policy_detects_operation_rules() { - let val = regorus::Value::from_json_str( - r#"{"protocol": "websocket", "host": "gateway.example.com", "port": 443, "rules": [{"allow": {"method": "GET", "path": "/graphql"}}, {"allow": {"operation_type": "subscription", "fields": ["messageAdded"]}}]}"#, - ) - .unwrap(); - let config = parse_l7_config(&val).unwrap(); - assert!(config.websocket_graphql_policy); + fn validate_mcp_wildcard_tool_requires_strict_tool_names() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "mcp.example.com", + "port": 443, + "path": "/mcp", + "protocol": "mcp", + "mcp_strict_tool_names": false, + "rules": [{ + "allow": { + "method": "tools/call", + "tool": "read_*" + } + }, { + "allow": { + "method": "tools/call", + "params": { + "name": { "any": ["safe_tool", "list_*"] } + } + } + }], + "deny_rules": [{ + "method": "tools/call", + "tool": "delete_resource" + }] + }], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| { + e.contains("rules[0].allow.tool") + && e.contains("strict_tool_names to remain enabled") + }), + "wildcard tool aliases should require strict tool names: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("rules[1].allow.params.name") + && e.contains("strict_tool_names to remain enabled") + }), + "wildcard params.name should require strict tool names: {errors:?}" + ); } #[test] - fn validate_websocket_credential_rewrite_warns_unless_rest_or_websocket() { + fn validate_mcp_protocol_requires_endpoint_target() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", - "port": 443, - "websocket_credential_rewrite": true + "protocol": "mcp" }], "binaries": [] } } }); - let (_errors, warnings) = validate_l7_policies(&data); + let (errors, _warnings) = validate_l7_policies(&data); assert!( - warnings + errors .iter() - .any(|w| w.contains("websocket_credential_rewrite is ignored")), - "expected websocket_credential_rewrite warning: {warnings:?}" + .any(|e| e.contains("protocol mcp requires host")), + "MCP protocol-only endpoint should require host: {errors:?}" + ); + assert!( + errors + .iter() + .any(|e| e.contains("protocol mcp requires port or ports")), + "MCP protocol-only endpoint should require port: {errors:?}" ); } #[test] - fn validate_request_body_credential_rewrite_warns_unless_rest() { + fn validate_mcp_broad_tools_call_deny_rejects_tool_allow_rules() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", + "host": "mcp.example.com", "port": 443, - "protocol": "websocket", - "request_body_credential_rewrite": true + "path": "/mcp", + "protocol": "mcp", + "rules": [{ + "allow": { + "method": "tools/call", + "tool": "read_status" + } + }], + "deny_rules": [{ + "method": "tools/call" + }] }], "binaries": [] } } }); - let (_errors, warnings) = validate_l7_policies(&data); + let (errors, _warnings) = validate_l7_policies(&data); assert!( - warnings - .iter() - .any(|w| w.contains("request_body_credential_rewrite is ignored")), - "expected request_body_credential_rewrite warning: {warnings:?}" + errors.iter().any(|e| { + e.contains("deny_rules[0]") + && e.contains("denies every tool call") + && e.contains("conflicts with MCP tool allow rules") + }), + "broad tools/call deny should reject tool allow rules with a reason: {errors:?}" ); } #[test] - fn expand_websocket_read_write_access_includes_text_messages() { - let mut data = serde_json::json!({ + fn validate_mcp_broad_tools_call_allow_rejects_tool_allow_rules() { + let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", + "host": "mcp.example.com", "port": 443, - "protocol": "websocket", - "access": "read-write" + "path": "/mcp", + "protocol": "mcp", + "rules": [{ + "allow": { + "method": "tools/*" + } + }, { + "allow": { + "tool": "read_status" + } + }] }], "binaries": [] } } }); - - expand_access_presets(&mut data); - let rules = data["network_policies"]["test"]["endpoints"][0]["rules"] - .as_array() - .unwrap(); - let methods: Vec<&str> = rules - .iter() - .map(|r| r["allow"]["method"].as_str().unwrap()) - .collect(); - assert!(methods.contains(&"GET")); - assert!(methods.contains(&"WEBSOCKET_TEXT")); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors.iter().any(|e| { + e.contains("rules[0].allow") + && e.contains("allows every tool call") + && e.contains("conflicts with MCP tool allow rules") + }), + "broad tools/call allow should reject tool allow rules with a reason: {errors:?}" + ); } #[test] - fn validate_websocket_accepts_graphql_operation_rules() { + fn validate_jsonrpc_rejects_params_matchers() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", + "host": "jsonrpc.example.com", "port": 443, - "protocol": "websocket", - "rules": [ - {"allow": {"method": "GET", "path": "/graphql"}}, - {"allow": {"operation_type": "subscription", "fields": ["messageAdded"]}} - ] + "path": "/rpc", + "protocol": "json-rpc", + "rules": [{ + "allow": { + "method": "reports.search", + "params": { "query": "quarterly" } + } + }], + "deny_rules": [{ + "method": "reports.archive", + "params": { "report_id": "rpt-123" } + }] }], "binaries": [] } } }); - let (errors, warnings) = validate_l7_policies(&data); - assert!(errors.is_empty(), "expected no errors: {errors:?}"); - assert!(warnings.is_empty(), "expected no warnings: {warnings:?}"); + let (errors, _warnings) = validate_l7_policies(&data); + assert!( + errors + .iter() + .any(|e| { e.contains("rules[0].allow") && e.contains("do not support params") }), + "JSON-RPC allow params should be rejected: {errors:?}" + ); + assert!( + errors + .iter() + .any(|e| { e.contains("deny_rules[0]") && e.contains("do not support params") }), + "JSON-RPC deny params should be rejected: {errors:?}" + ); } #[test] - fn validate_websocket_graphql_rule_requires_operation_type() { + fn validate_mcp_strict_tool_names_is_mcp_only() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [ + { + "host": "mcp.example.com", + "port": 443, + "path": "/mcp", + "protocol": "mcp", + "mcp_strict_tool_names": false, + "rules": [{ "allow": { "method": "tools/call" } }] + }, + { + "host": "api.example.com", + "port": 443, + "protocol": "rest", + "mcp_strict_tool_names": false, + "mcp_allow_all_known_mcp_methods": true, + "access": "full" + } + ], + "binaries": [] + } + } + }); + let (errors, _warnings) = validate_l7_policies(&data); + assert_eq!( + errors + .iter() + .filter(|error| error.contains("is only valid for protocol mcp")) + .count(), + 2, + "only the REST endpoint should reject MCP-specific options: {errors:?}" + ); + } + + #[test] + fn validate_mcp_options_require_bool() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", + "host": "mcp.example.com", "port": 443, - "protocol": "websocket", - "rules": [ - {"allow": {"method": "GET", "path": "/graphql"}}, - {"allow": {"fields": ["messageAdded"]}} - ] + "path": "/mcp", + "protocol": "mcp", + "mcp_strict_tool_names": "false", + "mcp_allow_all_known_mcp_methods": "false", + "rules": [{ "allow": { "method": "tools/call" } }] }], "binaries": [] } @@ -1519,24 +2450,29 @@ mod tests { }); let (errors, _warnings) = validate_l7_policies(&data); assert!( - errors.iter().any(|e| e.contains("operation_type")), - "expected missing operation_type error: {errors:?}" + errors + .iter() + .any(|error| error.contains("mcp.strict_tool_names must be boolean")), + "expected bool validation error: {errors:?}" + ); + assert!( + errors + .iter() + .any(|error| error.contains("mcp.allow_all_known_mcp_methods must be boolean")), + "expected bool validation error: {errors:?}" ); } #[test] - fn validate_websocket_graphql_rule_rejects_mixed_transport_fields() { + fn validate_mcp_requires_rules_by_default() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", + "host": "mcp.example.com", "port": 443, - "protocol": "websocket", - "rules": [ - {"allow": {"method": "GET", "path": "/graphql"}}, - {"allow": {"method": "WEBSOCKET_TEXT", "path": "/graphql", "operation_type": "subscription"}} - ] + "path": "/mcp", + "protocol": "mcp" }], "binaries": [] } @@ -1544,25 +2480,32 @@ mod tests { }); let (errors, _warnings) = validate_l7_policies(&data); assert!( - errors.iter().any(|e| e.contains("must not combine")), - "expected mixed-field error: {errors:?}" + errors.iter().any(|error| { + error.contains("protocol mcp requires rules") + && error.contains("mcp.allow_all_known_mcp_methods is false") + }), + "expected disabled method profile to require rules: {errors:?}" ); } #[test] - fn validate_websocket_graphql_policy_rejects_raw_text_message_rule() { + fn validate_jsonrpc_deny_rules_require_method() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "gateway.example.com", + "host": "jsonrpc.example.com", "port": 443, - "protocol": "websocket", - "rules": [ - {"allow": {"method": "GET", "path": "/graphql"}}, - {"allow": {"method": "WEBSOCKET_TEXT", "path": "/graphql"}}, - {"allow": {"operation_type": "query"}} - ] + "path": "/rpc", + "protocol": "json-rpc", + "rules": [{ + "allow": { + "method": "*" + } + }], + "deny_rules": [{ + "params": { "name": "delete_resource" } + }] }], "binaries": [] } @@ -1572,29 +2515,57 @@ mod tests { assert!( errors .iter() - .any(|e| e.contains("instead of WEBSOCKET_TEXT")), - "expected raw WEBSOCKET_TEXT rejection: {errors:?}" + .any(|e| e.contains("deny_rules[0].method") && e.contains("required")), + "JSON-RPC deny rules without method should be rejected: {errors:?}" ); } #[test] - fn validate_rules_and_access_mutual_exclusion() { + fn validate_jsonrpc_method_rejects_globs_except_allow_all() { let data = serde_json::json!({ "network_policies": { "test": { "endpoints": [{ - "host": "api.example.com", + "host": "jsonrpc.example.com", "port": 443, - "protocol": "rest", - "access": "read-only", - "rules": [{"allow": {"method": "GET", "path": "**"}}] + "path": "/rpc", + "protocol": "json-rpc", + "rules": [{ + "allow": { + "method": "*" + } + }, { + "allow": { + "method": "reports.*" + } + }], + "deny_rules": [{ + "method": "reports/{archive,delete}" + }] }], "binaries": [] } } }); let (errors, _warnings) = validate_l7_policies(&data); - assert!(errors.iter().any(|e| e.contains("mutually exclusive"))); + assert!( + !errors.iter().any(|e| e.contains("rules[0].allow.method")), + "JSON-RPC method: * should remain the allow-all sentinel: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("rules[1].allow.method") + && e.contains("do not support glob or wildcard matchers") + }), + "JSON-RPC allow method globs should be rejected: {errors:?}" + ); + assert!( + errors.iter().any(|e| { + e.contains("deny_rules[0].method") + && e.contains("do not support glob or wildcard matchers") + }), + "JSON-RPC deny method globs should be rejected: {errors:?}" + ); } #[test] @@ -2324,6 +3295,45 @@ mod tests { ); } + #[test] + fn validate_jsonrpc_nested_params_matchers_are_rejected() { + let data = serde_json::json!({ + "network_policies": { + "test": { + "endpoints": [{ + "host": "jsonrpc.example.com", + "port": 443, + "protocol": "json-rpc", + "rules": [{ + "allow": { + "method": "reports.search", + "params": { + "query": "quarterly", + "filters": { + "scope": "workspace/main", + "repository": { "any": ["NVIDIA/OpenShell", "NVIDIA/*"] } + } + } + } + }] + }], + "binaries": [] + } + } + }); + let (errors, warnings) = validate_l7_policies(&data); + assert!( + errors + .iter() + .any(|e| { e.contains("rules[0].allow") && e.contains("do not support params") }), + "JSON-RPC nested params matchers should be rejected: {errors:?}" + ); + assert!( + warnings.is_empty(), + "unsupported params should not emit warnings: {warnings:?}" + ); + } + // --- Deny rules validation tests --- #[test] @@ -2477,7 +3487,7 @@ mod tests { assert!( errors .iter() - .any(|e| e.contains("expected string glob or object")), + .any(|e| e.contains("expected string glob or matcher object")), "should reject non-string/non-object matcher in deny query: {errors:?}" ); } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 5c95760a1d..ed2bde1135 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -139,6 +139,8 @@ fn emit_parse_rejection(ctx: &L7EvalContext, detail: &str, engine_type: &str) { fn engine_type_for_protocol(protocol: L7Protocol) -> &'static str { match protocol { L7Protocol::Graphql => "l7-graphql", + L7Protocol::JsonRpc => "l7-jsonrpc", + L7Protocol::Mcp => "l7-mcp", L7Protocol::Websocket => "l7-websocket", L7Protocol::Rest | L7Protocol::Sql => "l7", } @@ -215,6 +217,9 @@ where .into_diagnostic()?; Ok(()) } + L7Protocol::JsonRpc | L7Protocol::Mcp => { + relay_jsonrpc(config, &engine, client, upstream, ctx).await + } } } @@ -308,6 +313,43 @@ where } else { None }; + let jsonrpc_info = if config.protocol.is_jsonrpc_family() { + if crate::l7::jsonrpc::jsonrpc_receive_stream_request(&req) { + Some(crate::l7::jsonrpc::JsonRpcRequestInfo::receive_stream()) + } else { + match crate::l7::http::read_body_for_inspection( + client, + &mut req, + config.json_rpc_max_body_bytes, + ) + .await + { + Ok(body) => Some(crate::l7::jsonrpc::parse_jsonrpc_body_with_options( + &body, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + )), + Err(e) => { + if is_benign_connection_error(&e) { + debug!( + host = %ctx.host, + port = ctx.port, + error = %e, + "JSON-RPC L7 connection closed" + ); + } else { + let detail = parse_rejection_detail( + &e.to_string(), + ParseRejectionMode::L7Endpoint, + ); + emit_parse_rejection(ctx, &detail, "l7-jsonrpc"); + } + return Ok(()); + } + } + } + } else { + None + }; if close_if_stale(engine.generation_guard(), ctx) { return Ok(()); @@ -338,6 +380,7 @@ where target: redacted_target.clone(), query_params: req.query_params.clone(), graphql: graphql_info.clone(), + jsonrpc: jsonrpc_info.clone(), }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { @@ -361,7 +404,13 @@ where let parse_error_reason = graphql_info .as_ref() .and_then(|info| info.error.as_deref()) - .map(|error| format!("GraphQL request rejected: {error}")); + .map(|error| format!("GraphQL request rejected: {error}")) + .or_else(|| { + jsonrpc_info + .as_ref() + .and_then(|info| info.error.as_deref()) + .map(|error| format!("JSON-RPC request rejected: {error}")) + }); let force_deny = parse_error_reason.is_some(); let (allowed, reason) = if let Some(reason) = parse_error_reason { (false, reason) @@ -382,8 +431,12 @@ where let engine_type = match config.protocol { L7Protocol::Graphql => "l7-graphql", L7Protocol::Websocket => "l7-websocket", + L7Protocol::JsonRpc => "l7-jsonrpc", + L7Protocol::Mcp => "l7-mcp", L7Protocol::Rest | L7Protocol::Sql => "l7", }; + let protocol_summary = + l7_protocol_log_summary(graphql_info.as_ref(), jsonrpc_info.as_ref()); emit_l7_request_log( ctx, &request_info, @@ -391,7 +444,7 @@ where decision_str, engine_type, &reason, - graphql_info.as_ref(), + &protocol_summary, ); let _ = &eval_target; @@ -474,7 +527,7 @@ fn emit_l7_request_log( decision_str: &str, engine_type: &str, reason: &str, - graphql_info: Option<&crate::l7::graphql::GraphqlRequestInfo>, + protocol_summary: &str, ) { let (action_id, disposition_id, severity) = match decision_str { "deny" => (ActionId::Denied, DispositionId::Blocked, SeverityId::Medium), @@ -489,9 +542,6 @@ fn emit_l7_request_log( SeverityId::Informational, ), }; - let summary = graphql_info - .map(|info| format!(" {}", graphql_log_summary(info))) - .unwrap_or_default(); let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) .action(action_id) @@ -505,13 +555,28 @@ fn emit_l7_request_log( .firewall_rule(&ctx.policy_name, engine_type) .message(format!( "L7_REQUEST {decision_str} {} {}:{}{}{} reason={}", - request_info.action, ctx.host, ctx.port, redacted_target, summary, reason, + request_info.action, ctx.host, ctx.port, redacted_target, protocol_summary, reason, )) .build(); ocsf_emit!(event); emit_activity(ctx, decision_str == "deny", "l7_policy"); } +fn l7_protocol_log_summary( + graphql_info: Option<&crate::l7::graphql::GraphqlRequestInfo>, + jsonrpc_info: Option<&crate::l7::jsonrpc::JsonRpcRequestInfo>, +) -> String { + if let Some(info) = graphql_info { + return format!(" {}", graphql_log_summary(info)); + } + + if let Some(info) = jsonrpc_info { + return format!(" rule_methods={}", rule_method_names_for_log(info)); + } + + String::new() +} + fn emit_activity(ctx: &L7EvalContext, denied: bool, deny_group: &'static str) { if let Some(tx) = &ctx.activity_tx { let _ = try_record_activity(tx, denied, deny_group); @@ -662,6 +727,13 @@ pub(crate) fn websocket_extension_mode(config: &L7EndpointConfig) -> WebSocketEx } } +fn jsonrpc_engine_type(protocol: L7Protocol) -> &'static str { + match protocol { + L7Protocol::Mcp => "l7-mcp", + _ => "l7-jsonrpc", + } +} + /// REST relay loop: parse request -> evaluate -> allow/deny -> relay response -> repeat. async fn relay_rest( config: &L7EndpointConfig, @@ -744,6 +816,7 @@ where target: redacted_target.clone(), query_params: req.query_params.clone(), graphql: None, + jsonrpc: None, }; let websocket_request = crate::l7::rest::request_is_websocket_upgrade(&req.raw_header); if config.protocol == L7Protocol::Websocket && !websocket_request { @@ -940,6 +1013,182 @@ fn close_if_stale(guard: &PolicyGenerationGuard, ctx: &L7EvalContext) -> bool { true } +async fn relay_jsonrpc( + config: &L7EndpointConfig, + engine: &TunnelPolicyEngine, + client: &mut C, + upstream: &mut U, + ctx: &L7EvalContext, +) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + loop { + if close_if_stale(engine.generation_guard(), ctx) { + return Ok(()); + } + + // Future MCP version-profile request checks should hook here before OPA + // evaluation. See McpOptions in proto/sandbox.proto for the policy + // roadmap and source documentation. + let parsed = match crate::l7::jsonrpc::parse_jsonrpc_http_request( + client, + config.json_rpc_max_body_bytes, + crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: config.allow_encoded_slash, + ..Default::default() + }, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(config), + ) + .await + { + Ok(Some(parsed)) => parsed, + Ok(None) => return Ok(()), + Err(e) => { + if is_benign_connection_error(&e) { + debug!( + host = %ctx.host, + port = ctx.port, + error = %e, + "JSON-RPC L7 connection closed" + ); + } else { + let detail = + parse_rejection_detail(&e.to_string(), ParseRejectionMode::L7Endpoint); + emit_parse_rejection(ctx, &detail, jsonrpc_engine_type(config.protocol)); + } + return Ok(()); + } + }; + + let req = parsed.request; + let jsonrpc_info = parsed.info; + + if close_if_stale(engine.generation_guard(), ctx) { + return Ok(()); + } + + let redacted_target = req.target.clone(); + + let request_info = L7RequestInfo { + action: req.action.clone(), + target: redacted_target.clone(), + query_params: req.query_params.clone(), + graphql: None, + jsonrpc: Some(jsonrpc_info.clone()), + }; + + let parse_error_reason = jsonrpc_info + .error + .as_deref() + .map(|e| format!("JSON-RPC request rejected: {e}")); + let response_frame_reason = + jsonrpc_response_frame_hard_deny_reason(config.protocol, &jsonrpc_info); + let force_deny = parse_error_reason.is_some() || response_frame_reason.is_some(); + let (allowed, reason, jsonrpc_log_info) = if let Some(reason) = parse_error_reason { + (false, reason, jsonrpc_info.clone()) + } else if let Some(reason) = response_frame_reason { + (false, reason, jsonrpc_info.clone()) + } else { + let evaluation = + evaluate_jsonrpc_l7_request_for_log(engine, ctx, &request_info, &jsonrpc_info)?; + (evaluation.allowed, evaluation.reason, evaluation.log_info) + }; + + if close_if_stale(engine.generation_guard(), ctx) { + return Ok(()); + } + + let decision_str = match (allowed, config.enforcement) { + (_, _) if force_deny => "deny", + (true, _) => "allow", + (false, EnforcementMode::Audit) => "audit", + (false, EnforcementMode::Enforce) => "deny", + }; + + { + let (action_id, disposition_id, severity) = match decision_str { + "deny" => (ActionId::Denied, DispositionId::Blocked, SeverityId::Medium), + _ => ( + ActionId::Allowed, + DispositionId::Allowed, + SeverityId::Informational, + ), + }; + let endpoint = format!("{}:{}{}", ctx.host, ctx.port, redacted_target); + let policy_version = engine.captured_generation(); + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(action_id) + .disposition(disposition_id) + .severity(severity) + .http_request(HttpRequest::new( + &request_info.action, + OcsfUrl::new("http", &ctx.host, &redacted_target, ctx.port), + )) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .firewall_rule(&ctx.policy_name, jsonrpc_engine_type(config.protocol)) + .message(jsonrpc_log_message( + decision_str, + &request_info.action, + &endpoint, + &jsonrpc_log_info, + policy_version, + &reason, + )) + .build(); + ocsf_emit!(event); + } + + if allowed || (config.enforcement == EnforcementMode::Audit && !force_deny) { + // Future MCP response/SSE introspection or rewrite would hook here + // before returning upstream bytes. The current policy schema has no + // trusted-annotations or version-profile field, so MCP responses and + // SSE streams are relayed unchanged; see McpOptions in + // proto/sandbox.proto for planned policy extensions. + let outcome = crate::l7::rest::relay_http_request_with_resolver_guarded( + &req, + client, + upstream, + ctx.secret_resolver.as_deref(), + Some(engine.generation_guard()), + ) + .await?; + match outcome { + RelayOutcome::Reusable => {} + RelayOutcome::Consumed => { + debug!( + host = %ctx.host, + port = ctx.port, + "Upstream connection not reusable, closing JSON-RPC L7 relay" + ); + return Ok(()); + } + RelayOutcome::Upgraded { .. } => { + return Ok(()); + } + } + } else { + crate::l7::rest::RestProvider::default() + .deny_with_redacted_target( + &req, + &ctx.policy_name, + &reason, + client, + Some(&redacted_target), + Some(crate::l7::rest::DenyResponseContext { + host: Some(&ctx.host), + port: Some(ctx.port), + binary: Some(&ctx.binary_path), + }), + ) + .await?; + return Ok(()); + } + } +} + async fn relay_graphql( config: &L7EndpointConfig, engine: &TunnelPolicyEngine, @@ -1021,6 +1270,7 @@ where target: redacted_target.clone(), query_params: req.query_params.clone(), graphql: Some(graphql_info.clone()), + jsonrpc: None, }; // Malformed or ambiguous GraphQL requests, such as duplicated GET @@ -1169,6 +1419,55 @@ fn graphql_log_summary(info: &crate::l7::graphql::GraphqlRequestInfo) -> String format!("graphql_ops={}", ops.join(";")) } +pub(crate) fn jsonrpc_log_message( + decision: &str, + http_method: &str, + endpoint: &str, + info: &crate::l7::jsonrpc::JsonRpcRequestInfo, + policy_version: u64, + reason: &str, +) -> String { + let rule_methods = rule_method_names_for_log(info); + format!( + "JSONRPC_L7_REQUEST decision={decision} http_method={http_method} endpoint={endpoint} rule_methods={rule_methods} policy_version={policy_version} reason={reason}" + ) +} + +pub(crate) fn rule_method_names_for_log(info: &crate::l7::jsonrpc::JsonRpcRequestInfo) -> String { + if info.calls.is_empty() { + return "-".to_string(); + } + info.calls + .iter() + .map(|call| sanitize_log_token(&call.method)) + .collect::>() + .join(",") +} + +fn sanitize_log_token(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_control() { '?' } else { ch }) + .collect() +} + +struct JsonRpcEvaluation { + allowed: bool, + reason: String, + log_info: crate::l7::jsonrpc::JsonRpcRequestInfo, +} + +pub(crate) const JSONRPC_RESPONSE_FRAME_DENY_REASON: &str = + "JSON-RPC response frames are not permitted from client to server"; + +pub(crate) fn jsonrpc_response_frame_hard_deny_reason( + protocol: L7Protocol, + jsonrpc: &crate::l7::jsonrpc::JsonRpcRequestInfo, +) -> Option { + (protocol != L7Protocol::Mcp && jsonrpc.has_response) + .then(|| JSONRPC_RESPONSE_FRAME_DENY_REASON.to_string()) +} + /// Check if a miette error represents a benign connection close. /// /// TLS handshake EOF, missing `close_notify`, connection resets, and broken @@ -1194,6 +1493,109 @@ pub fn evaluate_l7_request( engine: &TunnelPolicyEngine, ctx: &L7EvalContext, request: &L7RequestInfo, +) -> Result<(bool, String)> { + if let Some(jsonrpc) = &request.jsonrpc + && jsonrpc.is_batch + && !jsonrpc.calls.is_empty() + { + if jsonrpc.has_response { + let (allowed, reason) = evaluate_l7_request_once(engine, ctx, request)?; + if !allowed { + return Ok((false, reason)); + } + } + for call in &jsonrpc.calls { + let item_request = jsonrpc_request_for_call(request, call); + let (allowed, reason) = evaluate_l7_request_once(engine, ctx, &item_request)?; + if !allowed { + return Ok((false, reason)); + } + } + return Ok((true, String::new())); + } + + evaluate_l7_request_once(engine, ctx, request) +} + +fn evaluate_jsonrpc_l7_request_for_log( + engine: &TunnelPolicyEngine, + ctx: &L7EvalContext, + request: &L7RequestInfo, + jsonrpc: &crate::l7::jsonrpc::JsonRpcRequestInfo, +) -> Result { + if jsonrpc.has_response { + let (allowed, reason) = evaluate_l7_request_once(engine, ctx, request)?; + if !allowed || !jsonrpc.is_batch || jsonrpc.calls.is_empty() { + return Ok(JsonRpcEvaluation { + allowed, + reason, + log_info: jsonrpc.clone(), + }); + } + } + + if jsonrpc.is_batch && !jsonrpc.calls.is_empty() { + let mut denied_calls = Vec::new(); + let mut first_denied_reason = None; + for call in &jsonrpc.calls { + let item_request = jsonrpc_request_for_call(request, call); + let (allowed, reason) = evaluate_l7_request_once(engine, ctx, &item_request)?; + if !allowed { + if first_denied_reason.is_none() { + first_denied_reason = Some(reason); + } + denied_calls.push(call.clone()); + } + } + + if denied_calls.is_empty() { + return Ok(JsonRpcEvaluation { + allowed: true, + reason: String::new(), + log_info: jsonrpc.clone(), + }); + } + + return Ok(JsonRpcEvaluation { + allowed: false, + reason: first_denied_reason.unwrap_or_else(|| "request denied by policy".to_string()), + log_info: crate::l7::jsonrpc::JsonRpcRequestInfo { + calls: denied_calls, + is_batch: true, + receive_stream: false, + has_response: false, + error: None, + }, + }); + } + + let (allowed, reason) = evaluate_l7_request_once(engine, ctx, request)?; + Ok(JsonRpcEvaluation { + allowed, + reason, + log_info: jsonrpc.clone(), + }) +} + +fn jsonrpc_request_for_call( + request: &L7RequestInfo, + call: &crate::l7::jsonrpc::JsonRpcCallInfo, +) -> L7RequestInfo { + let mut item_request = request.clone(); + item_request.jsonrpc = Some(crate::l7::jsonrpc::JsonRpcRequestInfo { + calls: vec![call.clone()], + is_batch: false, + receive_stream: false, + has_response: false, + error: None, + }); + item_request +} + +fn evaluate_l7_request_once( + engine: &TunnelPolicyEngine, + ctx: &L7EvalContext, + request: &L7RequestInfo, ) -> Result<(bool, String)> { if engine.is_stale() { return Err(miette!( @@ -1218,6 +1620,17 @@ pub fn evaluate_l7_request( "path": request.target, "query_params": request.query_params.clone(), "graphql": request.graphql.clone(), + "jsonrpc": request.jsonrpc.as_ref().map(|j| { + let call = if j.is_batch { None } else { j.calls.first() }; + serde_json::json!({ + "method": call.map(|call| call.method.as_str()), + "params": call.map(|call| &call.params), + "tool": call.and_then(|call| call.tool.as_deref()), + "receive_stream": j.receive_stream, + "has_response": j.has_response, + "error": j.error, + }) + }), } }); @@ -1531,25 +1944,117 @@ network_policies: (generation_guard, ctx, fixture) } - fn authorization_header_count(headers: &str) -> usize { - headers - .lines() - .filter(|line| { - line.split_once(':') - .is_some_and(|(name, _)| name.eq_ignore_ascii_case("authorization")) - }) - .count() - } - - #[test] - fn parse_rejection_detail_adds_l7_hint_for_encoded_slash() { - let detail = parse_rejection_detail( - "HTTP request-target rejected: request-target contains an encoded '/' (%2F) which is not allowed on this endpoint", - ParseRejectionMode::L7Endpoint, - ); - - assert!(detail.contains("allow_encoded_slash: true")); - assert!(detail.contains("upstream requires encoded slashes")); + fn jsonrpc_test_relay_context() -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { + let data = r" +network_policies: + jsonrpc_api: + name: jsonrpc_api + endpoints: + - host: jsonrpc.example.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/python3 } +"; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "jsonrpc.example.test".into(), + port: 8000, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&endpoint_config.unwrap()).unwrap(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "jsonrpc.example.test".into(), + port: 8000, + policy_name: "jsonrpc_api".into(), + binary_path: "/usr/bin/python3".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + (config, tunnel_engine, ctx) + } + + fn mcp_test_relay_context() -> (L7EndpointConfig, TunnelPolicyEngine, L7EvalContext) { + let data = r" +network_policies: + mcp_api: + name: mcp_api + endpoints: + - host: mcp.example.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/python3 } +"; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let input = NetworkInput { + host: "mcp.example.test".into(), + port: 8000, + binary_path: PathBuf::from("/usr/bin/python3"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (endpoint_config, generation) = engine + .query_endpoint_config_with_generation(&input) + .unwrap(); + let config = crate::l7::parse_l7_config(&endpoint_config.unwrap()).unwrap(); + let tunnel_engine = engine.clone_engine_for_tunnel(generation).unwrap(); + let ctx = L7EvalContext { + host: "mcp.example.test".into(), + port: 8000, + policy_name: "mcp_api".into(), + binary_path: "/usr/bin/python3".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + (config, tunnel_engine, ctx) + } + + fn authorization_header_count(headers: &str) -> usize { + headers + .lines() + .filter(|line| { + line.split_once(':') + .is_some_and(|(name, _)| name.eq_ignore_ascii_case("authorization")) + }) + .count() + } + + #[test] + fn parse_rejection_detail_adds_l7_hint_for_encoded_slash() { + let detail = parse_rejection_detail( + "HTTP request-target rejected: request-target contains an encoded '/' (%2F) which is not allowed on this endpoint", + ParseRejectionMode::L7Endpoint, + ); + + assert!(detail.contains("allow_encoded_slash: true")); + assert!(detail.contains("upstream requires encoded slashes")); } #[test] @@ -1851,6 +2356,7 @@ network_policies: target: "/ws".into(), query_params: std::collections::HashMap::new(), graphql: None, + jsonrpc: None, }; let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); @@ -1859,6 +2365,325 @@ network_policies: assert!(reason.contains("WEBSOCKET_TEXT /ws not permitted")); } + #[test] + fn jsonrpc_batch_evaluates_each_call() { + let data = r#" +network_policies: + jsonrpc_api: + name: jsonrpc_api + endpoints: + - host: api.example.test + port: 443 + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: "reports.list" + - allow: + method: "reports.search" + deny_rules: + - method: "reports.delete" + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let tunnel_engine = engine + .clone_engine_for_tunnel(engine.current_generation()) + .unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "jsonrpc_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let mut request = L7RequestInfo { + action: "POST".into(), + target: "/rpc".into(), + query_params: std::collections::HashMap::new(), + graphql: None, + jsonrpc: Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"method":"reports.search","params":{"query":"private_query_value"}} + ]"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + )), + }; + + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(allowed, "{reason}"); + + request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"result":{"ok":true}} + ]"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + )); + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(!allowed); + assert!(reason.contains("response frames")); + + let jsonrpc = request.jsonrpc.as_ref().expect("jsonrpc request"); + let evaluation = + evaluate_jsonrpc_l7_request_for_log(&tunnel_engine, &ctx, &request, jsonrpc).unwrap(); + assert!(!evaluation.allowed); + assert!(evaluation.log_info.has_response); + assert_eq!( + rule_method_names_for_log(&evaluation.log_info), + "reports.list" + ); + + request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":2,"result":{"ok":true}}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + )); + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(!allowed); + assert!(reason.contains("response frames")); + + let jsonrpc = request.jsonrpc.as_ref().expect("jsonrpc response"); + let evaluation = + evaluate_jsonrpc_l7_request_for_log(&tunnel_engine, &ctx, &request, jsonrpc).unwrap(); + assert!(!evaluation.allowed); + assert!(evaluation.log_info.has_response); + assert_eq!(rule_method_names_for_log(&evaluation.log_info), "-"); + + request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"method":"reports.search","params":{"query":"private_query_value"}}, + {"jsonrpc":"2.0","id":3,"method":"reports.delete","params":{"id":"purge_cache"}} + ]"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + )); + let (allowed, _) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(!allowed); + + let jsonrpc = request.jsonrpc.as_ref().expect("jsonrpc request"); + let evaluation = + evaluate_jsonrpc_l7_request_for_log(&tunnel_engine, &ctx, &request, jsonrpc).unwrap(); + assert!(!evaluation.allowed); + assert!(evaluation.log_info.is_batch); + assert_eq!( + rule_method_names_for_log(&evaluation.log_info), + "reports.delete" + ); + + let message = jsonrpc_log_message( + "deny", + "POST", + "api.example.test:443/rpc", + &evaluation.log_info, + 42, + &evaluation.reason, + ); + assert!(message.contains("rule_methods=reports.delete")); + assert!(message.contains("policy_version=42")); + assert!(!message.contains("reports.list")); + assert!(!message.contains("reports.search")); + assert!(!message.contains("private_query_value")); + assert!(!message.contains("purge_cache")); + } + + #[test] + fn jsonrpc_request_params_do_not_affect_method_policy() { + let data = r#" +network_policies: + jsonrpc_api: + name: jsonrpc_api + endpoints: + - host: api.example.test + port: 443 + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: "reports.search" + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let tunnel_engine = engine + .clone_engine_for_tunnel(engine.current_generation()) + .unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "jsonrpc_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let mut request = L7RequestInfo { + action: "POST".into(), + target: "/rpc".into(), + query_params: std::collections::HashMap::new(), + graphql: None, + jsonrpc: Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":{"query":"delete_resource","filters":{"scope":"workspace/secret"}}}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + )), + }; + + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(allowed, "{reason}"); + + request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":["ignored",{"nested":true}]}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + )); + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(allowed, "{reason}"); + } + + #[test] + fn mcp_tool_deny_rule_blocks_tools_call() { + let data = r#" +network_policies: + mcp_api: + name: mcp_api + endpoints: + - host: api.example.test + port: 443 + path: "/mcp" + protocol: mcp + enforcement: enforce + mcp: + max_body_bytes: 131072 + rules: + - allow: + method: initialize + - allow: + method: tools/list + - allow: + method: tools/call + tool: read_status + deny_rules: + - method: tools/call + tool: delete_resource + binaries: + - { path: /usr/bin/node } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).unwrap(); + let tunnel_engine = engine + .clone_engine_for_tunnel(engine.current_generation()) + .unwrap(); + let ctx = L7EvalContext { + host: "api.example.test".into(), + port: 443, + policy_name: "mcp_api".into(), + binary_path: "/usr/bin/node".into(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + }; + let mut request = L7RequestInfo { + action: "POST".into(), + target: "/mcp".into(), + query_params: std::collections::HashMap::new(), + graphql: None, + jsonrpc: Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"read_status","arguments":{}}}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::Mcp, + )), + }; + + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(allowed, "{reason}"); + + request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"delete_resource","arguments":{"scope":"workspace/main"}}}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::Mcp, + )); + let parsed = request.jsonrpc.as_ref().expect("parsed MCP request"); + assert!( + parsed.error.is_none(), + "MCP request should parse: {parsed:?}" + ); + assert_eq!( + parsed.calls.first().and_then(|call| call.tool.as_deref()), + Some("delete_resource") + ); + + let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); + assert!(!allowed, "delete_resource must match the MCP deny rule"); + assert!( + reason.contains("deny rule"), + "deny reason should identify policy denial: {reason}" + ); + } + + #[test] + fn jsonrpc_log_records_method_names_not_params() { + let info = crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":1,"method":"reports.archive","params":{"id":"delete_resource","filters":{"scope":"secret-scope"}}}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + ); + let message = jsonrpc_log_message( + "deny", + "POST", + "jsonrpc.example.com:443/rpc", + &info, + 42, + "request denied by policy", + ); + + assert!(message.contains("endpoint=jsonrpc.example.com:443/rpc")); + assert!(message.contains("rule_methods=reports.archive")); + assert!(message.contains("policy_version=42")); + assert!(!message.contains("delete_resource")); + assert!(!message.contains("secret-scope")); + + let batch = crate::l7::jsonrpc::parse_jsonrpc_body( + br#"[ + {"jsonrpc":"2.0","id":1,"method":"reports.list"}, + {"jsonrpc":"2.0","id":2,"method":"reports.archive","params":{"id":"delete_resource"}} + ]"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + ); + let batch_message = jsonrpc_log_message( + "allow", + "POST", + "jsonrpc.example.com:443/rpc", + &batch, + 43, + "", + ); + + assert!(batch_message.starts_with("JSONRPC_L7_REQUEST ")); + assert!(batch_message.contains("rule_methods=reports.list,reports.archive")); + assert!(batch_message.contains("policy_version=43")); + assert!(!batch_message.contains("delete_resource")); + + let no_params = crate::l7::jsonrpc::parse_jsonrpc_body( + br#"{"jsonrpc":"2.0","id":1,"method":"initialize"}"#, + crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, + ); + let no_params_message = jsonrpc_log_message( + "allow", + "POST", + "jsonrpc.example.com:443/rpc", + &no_params, + 44, + "", + ); + assert!(no_params_message.contains("rule_methods=initialize")); + } + #[tokio::test] async fn route_selected_websocket_upgrade_rejects_invalid_accept_without_forwarding_101() { let data = r#" @@ -1887,6 +2712,8 @@ network_policies: tls: crate::l7::TlsMode::Auto, enforcement: EnforcementMode::Enforce, graphql_max_body_bytes: 0, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, allow_encoded_slash: false, websocket_credential_rewrite: true, request_body_credential_rewrite: false, @@ -1993,6 +2820,8 @@ network_policies: tls: crate::l7::TlsMode::Auto, enforcement: EnforcementMode::Enforce, graphql_max_body_bytes: 0, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, allow_encoded_slash: false, websocket_credential_rewrite: true, request_body_credential_rewrite: false, @@ -2116,6 +2945,8 @@ network_policies: tls: crate::l7::TlsMode::Auto, enforcement: EnforcementMode::Enforce, graphql_max_body_bytes: 0, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, allow_encoded_slash: false, websocket_credential_rewrite: true, request_body_credential_rewrite: false, @@ -2475,4 +3306,180 @@ network_policies: "stale passthrough request must not be forwarded upstream" ); } + + #[tokio::test] + async fn jsonrpc_relay_forwards_allowed_method() { + let (config, tunnel_engine, ctx) = jsonrpc_test_relay_context(); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"jsonrpc":"2.0","id":1,"method":"initialize"}"#; + let request = format!( + "POST /rpc HTTP/1.1\r\nHost: jsonrpc.example.test:8000\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + app.write_all(request.as_bytes()).await.unwrap(); + app.write_all(body).await.unwrap(); + + let mut upstream_bytes = Vec::new(); + let mut upstream_buf = [0u8; 1024]; + loop { + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_buf), + ) + .await + .expect("allowed JSON-RPC request should reach upstream") + .unwrap(); + assert_ne!(n, 0, "upstream closed before JSON-RPC body arrived"); + upstream_bytes.extend_from_slice(&upstream_buf[..n]); + if String::from_utf8_lossy(&upstream_bytes).contains(r#""method":"initialize""#) { + break; + } + } + let upstream_request = String::from_utf8_lossy(&upstream_bytes); + assert!(upstream_request.starts_with("POST /rpc HTTP/1.1")); + assert!(upstream_request.contains(r#""method":"initialize""#)); + + upstream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 36\r\nConnection: close\r\n\r\n{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}", + ) + .await + .unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("upstream response should reach client") + .unwrap(); + assert!(String::from_utf8_lossy(&response[..n]).contains("200 OK")); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should complete") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn mcp_relay_forwards_jsonrpc_response_frame() { + let (config, tunnel_engine, ctx) = mcp_test_relay_context(); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = br#"{"jsonrpc":"2.0","id":7,"result":{"action":"accept","content":{}}}"#; + let request = format!( + "POST /mcp HTTP/1.1\r\nHost: mcp.example.test:8000\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + app.write_all(request.as_bytes()).await.unwrap(); + app.write_all(body).await.unwrap(); + + let mut upstream_buf = [0u8; 1024]; + let n = tokio::time::timeout( + std::time::Duration::from_secs(1), + upstream.read(&mut upstream_buf), + ) + .await + .expect("MCP response frame should reach upstream") + .unwrap(); + let upstream_request = String::from_utf8_lossy(&upstream_buf[..n]); + assert!(upstream_request.starts_with("POST /mcp HTTP/1.1")); + assert!(upstream_request.contains(r#""result":{"action":"accept""#)); + + upstream + .write_all(b"HTTP/1.1 202 Accepted\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + .await + .unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(1), app.read(&mut response)) + .await + .expect("upstream response should reach client") + .unwrap(); + assert!(String::from_utf8_lossy(&response[..n]).contains("202 Accepted")); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should complete") + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn jsonrpc_relay_denies_method_not_in_allow_list() { + let (config, tunnel_engine, ctx) = jsonrpc_test_relay_context(); + let (mut app, mut relay_client) = tokio::io::duplex(8192); + let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); + let relay = tokio::spawn(async move { + relay_with_inspection( + &config, + tunnel_engine, + &mut relay_client, + &mut relay_upstream, + &ctx, + ) + .await + }); + + let body = + br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":{"query":"list_repos"}}"#; + let request = format!( + "POST /rpc HTTP/1.1\r\nHost: jsonrpc.example.test:8000\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + app.write_all(request.as_bytes()).await.unwrap(); + app.write_all(body).await.unwrap(); + + let mut response = [0u8; 512]; + let n = tokio::time::timeout(std::time::Duration::from_secs(2), app.read(&mut response)) + .await + .expect("relay should respond without reaching upstream") + .unwrap(); + let response = String::from_utf8_lossy(&response[..n]); + assert!( + response.contains("403"), + "reports.search not in allow list must be denied with 403, got: {response:?}" + ); + + let mut upstream_buf = [0u8; 128]; + let n = tokio::time::timeout( + std::time::Duration::from_millis(100), + upstream.read(&mut upstream_buf), + ) + .await + .unwrap_or(Ok(0)) + .unwrap_or(0); + assert_eq!(n, 0, "denied request must not be forwarded to upstream"); + + drop(app); + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("relay should complete") + .unwrap() + .unwrap(); + } } diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 65fd09fb6f..0558a67e55 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -1955,6 +1955,7 @@ where let header_str = String::from_utf8_lossy(&buf[..header_end]); let status_code = parse_status_code(&header_str).unwrap_or(200); let server_wants_close = parse_connection_close(&header_str); + let event_stream = response_is_event_stream(&header_str); let body_length = parse_body_length(&header_str)?; debug!( @@ -2014,19 +2015,29 @@ where // No explicit framing (no Content-Length, no Transfer-Encoding). // Per RFC 7230 §3.3.3 the body is delimited by connection close. if matches!(body_length, BodyLength::None) { - if server_wants_close { - // Server indicated it will close — read until EOF. + if server_wants_close || event_stream { + // Server indicated it will close, or this is a streaming response + // such as SSE where the body is intentionally delimited by EOF. let before_end = &buf[..header_end - 2]; client.write_all(before_end).await.into_diagnostic()?; - client - .write_all(b"Connection: close\r\n\r\n") - .await - .into_diagnostic()?; + if server_wants_close { + client + .write_all(b"Connection: close\r\n\r\n") + .await + .into_diagnostic()?; + } else { + client.write_all(b"\r\n").await.into_diagnostic()?; + } let overflow = &buf[header_end..]; if !overflow.is_empty() { client.write_all(overflow).await.into_diagnostic()?; + client.flush().await.into_diagnostic()?; + } + if event_stream { + relay_until_eof_without_idle_timeout(upstream, client).await?; + } else { + relay_until_eof(upstream, client).await?; } - relay_until_eof(upstream, client).await?; client.flush().await.into_diagnostic()?; return Ok(RelayOutcome::Consumed); } @@ -2096,6 +2107,19 @@ fn parse_connection_close(headers: &str) -> bool { false } +fn response_is_event_stream(headers: &str) -> bool { + headers.lines().skip(1).any(|line| { + let lower = line.to_ascii_lowercase(); + let Some(value) = lower.strip_prefix("content-type:") else { + return false; + }; + value + .split(';') + .next() + .is_some_and(|mime| mime.trim() == "text/event-stream") + }) +} + fn validate_websocket_response( headers: &str, mode: WebSocketExtensionMode, @@ -2300,7 +2324,10 @@ where loop { match tokio::time::timeout(RELAY_EOF_IDLE_TIMEOUT, reader.read(&mut buf)).await { Ok(Ok(0)) => return Ok(()), - Ok(Ok(n)) => writer.write_all(&buf[..n]).await.into_diagnostic()?, + Ok(Ok(n)) => { + writer.write_all(&buf[..n]).await.into_diagnostic()?; + writer.flush().await.into_diagnostic()?; + } Ok(Err(e)) => return Err(miette::miette!("{e}")), Err(_) => { debug!( @@ -2313,6 +2340,26 @@ where } } +/// Relay all bytes from reader to writer until EOF without an idle timeout. +/// +/// Used for server-sent events, where long idle gaps are part of the protocol +/// and do not mean the response body is complete. +async fn relay_until_eof_without_idle_timeout(reader: &mut R, writer: &mut W) -> Result<()> +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin, +{ + let mut buf = [0u8; RELAY_BUF_SIZE]; + loop { + let n = reader.read(&mut buf).await.into_diagnostic()?; + if n == 0 { + return Ok(()); + } + writer.write_all(&buf[..n]).await.into_diagnostic()?; + writer.flush().await.into_diagnostic()?; + } +} + /// Detect if the first bytes look like an HTTP request. /// /// Checks for common HTTP methods at the start of the stream. @@ -3337,6 +3384,19 @@ mod tests { )); } + #[test] + fn test_response_is_event_stream() { + assert!(response_is_event_stream( + "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\r\n" + )); + assert!(response_is_event_stream( + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream; charset=utf-8\r\n\r\n" + )); + assert!(!response_is_event_stream( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n" + )); + } + #[test] fn test_is_bodiless_response() { assert!(is_bodiless_response("HEAD", 200)); @@ -3394,6 +3454,99 @@ mod tests { ); } + #[tokio::test] + async fn relay_response_no_framing_event_stream_reads_until_eof() { + let response = + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\r\nevent: message\ndata: {}\r\n\r\n"; + + let (mut upstream_read, mut upstream_write) = tokio::io::duplex(4096); + let (mut client_read, mut client_write) = tokio::io::duplex(4096); + + tokio::spawn(async move { + upstream_write.write_all(response).await.unwrap(); + upstream_write.shutdown().await.unwrap(); + }); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(2), + relay_response( + "GET", + &mut upstream_read, + &mut client_write, + RelayResponseOptions::default(), + ), + ) + .await + .expect("relay_response should not deadlock"); + + let outcome = result.expect("relay_response should succeed"); + assert!( + matches!(outcome, RelayOutcome::Consumed), + "event stream is consumed by read-until-EOF" + ); + + client_write.shutdown().await.unwrap(); + let mut received = Vec::new(); + client_read.read_to_end(&mut received).await.unwrap(); + let received_str = String::from_utf8_lossy(&received); + assert!(received_str.contains("Content-Type: text/event-stream")); + assert!(received_str.contains("event: message")); + } + + #[tokio::test] + async fn relay_response_no_framing_event_stream_survives_idle_gap() { + let (mut upstream_read, mut upstream_write) = tokio::io::duplex(4096); + let (mut client_read, mut client_write) = tokio::io::duplex(4096); + + let upstream_task = tokio::spawn(async move { + upstream_write + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Type: text/event-stream; charset=utf-8\r\n\r\n", + ) + .await + .unwrap(); + upstream_write + .write_all(b"event: first\ndata: {}\r\n\r\n") + .await + .unwrap(); + upstream_write.flush().await.unwrap(); + tokio::time::sleep(RELAY_EOF_IDLE_TIMEOUT + std::time::Duration::from_secs(1)).await; + let _ = upstream_write + .write_all(b"event: second\ndata: {}\r\n\r\n") + .await; + let _ = upstream_write.shutdown().await; + }); + + let result = tokio::time::timeout( + RELAY_EOF_IDLE_TIMEOUT + std::time::Duration::from_secs(3), + relay_response( + "GET", + &mut upstream_read, + &mut client_write, + RelayResponseOptions::default(), + ), + ) + .await + .expect("event stream relay must outlive the generic EOF idle timeout"); + + let outcome = result.expect("relay_response should succeed"); + assert!( + matches!(outcome, RelayOutcome::Consumed), + "event stream is consumed by read-until-EOF" + ); + upstream_task.await.expect("upstream task should complete"); + + client_write.shutdown().await.unwrap(); + let mut received = Vec::new(); + client_read.read_to_end(&mut received).await.unwrap(); + let received_str = String::from_utf8_lossy(&received); + assert!(received_str.contains("event: first")); + assert!( + received_str.contains("event: second"), + "SSE event after idle gap should be forwarded, got: {received_str}" + ); + } + #[tokio::test] async fn relay_response_no_framing_without_connection_close_treats_as_empty() { // Response without Content-Length, TE, or Connection: close. diff --git a/crates/openshell-supervisor-network/src/l7/websocket.rs b/crates/openshell-supervisor-network/src/l7/websocket.rs index 31aa355090..e1f92e6ece 100644 --- a/crates/openshell-supervisor-network/src/l7/websocket.rs +++ b/crates/openshell-supervisor-network/src/l7/websocket.rs @@ -545,6 +545,7 @@ fn inspect_websocket_text_message( target: inspector.target.clone(), query_params: inspector.query_params.clone(), graphql: None, + jsonrpc: None, }; let (allowed, reason) = evaluate_l7_request(inspector.engine, inspector.ctx, &request_info)?; let decision = match (allowed, inspector.enforcement) { @@ -581,6 +582,7 @@ fn inspect_graphql_websocket_message( target: inspector.target.clone(), query_params: inspector.query_params.clone(), graphql: None, + jsonrpc: None, }; emit_websocket_l7_event( host, @@ -602,6 +604,7 @@ fn inspect_graphql_websocket_message( target: inspector.target.clone(), query_params: inspector.query_params.clone(), graphql: Some(graphql.clone()), + jsonrpc: None, }; let parse_error_reason = graphql .error diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 0e97f58571..fbab5fedd7 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -12,6 +12,7 @@ use openshell_core::policy::{ FilesystemPolicy, LandlockCompatibility, LandlockPolicy, ProcessPolicy, }; use openshell_core::proto::SandboxPolicy as ProtoSandboxPolicy; +use openshell_policy::L7ConfigStanza; use std::path::{Path, PathBuf}; use std::sync::{ Arc, Mutex, @@ -218,6 +219,8 @@ impl OpaEngine { )); } + normalize_l7_policy_rule_aliases(&mut data); + // Expand access presets to explicit rules after validation crate::l7::expand_access_presets(&mut data); @@ -723,6 +726,13 @@ fn preprocess_yaml_data(yaml_str: &str) -> Result { // Normalize port → ports for all endpoints so Rego always sees "ports" array. normalize_endpoint_ports(&mut data); + let config_errors = normalize_l7_config_aliases(&mut data); + if !config_errors.is_empty() { + return Err(miette::miette!( + "L7 policy validation failed:\n{}", + config_errors.join("\n") + )); + } // Validate BEFORE expanding presets (catches user errors like rules+access) let (errors, warnings) = crate::l7::validate_l7_policies(&data); @@ -744,6 +754,8 @@ fn preprocess_yaml_data(yaml_str: &str) -> Result { )); } + normalize_l7_policy_rule_aliases(&mut data); + // Expand access presets to explicit rules after validation crate::l7::expand_access_presets(&mut data); @@ -799,6 +811,150 @@ fn normalize_endpoint_ports(data: &mut serde_json::Value) { } } +fn normalize_l7_config_aliases(data: &mut serde_json::Value) -> Vec { + let mut errors = Vec::new(); + let Some(policies) = data + .get_mut("network_policies") + .and_then(|v| v.as_object_mut()) + else { + return errors; + }; + + for (policy_name, policy) in policies.iter_mut() { + let Some(endpoints) = policy.get_mut("endpoints").and_then(|v| v.as_array_mut()) else { + continue; + }; + + for (index, ep) in endpoints.iter_mut().enumerate() { + let Some(ep_obj) = ep.as_object_mut() else { + continue; + }; + let loc = format!("network_policies.{policy_name}.endpoints[{index}]"); + for stanza in L7ConfigStanza::ALL { + normalize_l7_config_alias(&mut errors, ep_obj, &loc, stanza); + } + } + } + + errors +} + +fn normalize_l7_config_alias( + errors: &mut Vec, + ep: &mut serde_json::Map, + loc: &str, + stanza: L7ConfigStanza, +) { + let key = stanza.key(); + let Some(config) = ep.get(key).cloned() else { + return; + }; + if config.is_null() { + ep.remove(key); + return; + } + match openshell_policy::l7_config_alias_runtime_fields(stanza, config) { + Ok(fields) => { + ep.remove(key); + for (field, value) in fields { + ep.entry(field.to_string()).or_insert(value); + } + } + Err(error) => errors.push(format!("{loc}.{key}: {error}")), + } +} + +fn normalize_l7_policy_rule_aliases(data: &mut serde_json::Value) { + let Some(policies) = data + .get_mut("network_policies") + .and_then(|v| v.as_object_mut()) + else { + return; + }; + + for (_name, policy) in policies.iter_mut() { + let Some(endpoints) = policy.get_mut("endpoints").and_then(|v| v.as_array_mut()) else { + continue; + }; + + for ep in endpoints.iter_mut() { + let Some(ep_obj) = ep.as_object_mut() else { + continue; + }; + normalize_l7_rules_aliases(ep_obj); + } + } +} + +fn normalize_l7_rules_aliases(ep: &mut serde_json::Map) { + let protocol = ep + .get("protocol") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .to_string(); + let mcp_allow_all_known_mcp_methods = ep + .get("mcp_allow_all_known_mcp_methods") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + if let Some(rules) = ep.get_mut("rules").and_then(|v| v.as_array_mut()) { + for rule in rules { + if let Some(allow) = rule + .get_mut("allow") + .and_then(serde_json::Value::as_object_mut) + { + normalize_l7_rule_aliases(allow, &protocol, mcp_allow_all_known_mcp_methods); + } else if let Some(allow) = rule.as_object_mut() { + normalize_l7_rule_aliases(allow, &protocol, mcp_allow_all_known_mcp_methods); + } + } + } + + if let Some(denies) = ep.get_mut("deny_rules").and_then(|v| v.as_array_mut()) { + for deny in denies { + if let Some(deny_obj) = deny.as_object_mut() { + normalize_l7_rule_aliases(deny_obj, &protocol, mcp_allow_all_known_mcp_methods); + } + } + } +} + +fn normalize_l7_rule_aliases( + rule: &mut serde_json::Map, + protocol: &str, + mcp_allow_all_known_mcp_methods: bool, +) { + if protocol == "mcp" { + let mut has_tool_selector = rule + .get("params") + .and_then(serde_json::Value::as_object) + .and_then(|params| params.get("name")) + .is_some_and(|v| !v.is_null()); + if let Some(tool) = rule.remove("tool").filter(|v| !v.is_null()) { + let params = rule + .entry("params".to_string()) + .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new())); + if let Some(params) = params.as_object_mut() { + params.entry("name".to_string()).or_insert(tool); + has_tool_selector = true; + } + } + + if mcp_allow_all_known_mcp_methods + && rule + .get("method") + .and_then(serde_json::Value::as_str) + .unwrap_or("") + .is_empty() + { + let method = if has_tool_selector { "tools/call" } else { "*" }; + rule.insert( + "method".to_string(), + serde_json::Value::String(method.to_string()), + ); + } + } +} + /// Resolve a policy binary path through the container's root filesystem. /// /// On Linux, `/proc//root/` provides access to the container's mount @@ -925,6 +1081,24 @@ fn resolve_binary_in_container(_policy_path: &str, _entrypoint_pid: u32) -> Opti None } +fn l7_matchers_to_json( + matchers: &std::collections::HashMap, +) -> serde_json::Map { + matchers + .iter() + .map(|(key, matcher)| { + let mut matcher_json = serde_json::json!({}); + if !matcher.glob.is_empty() { + matcher_json["glob"] = matcher.glob.clone().into(); + } + if !matcher.any.is_empty() { + matcher_json["any"] = matcher.any.clone().into(); + } + (key.clone(), matcher_json) + }) + .collect() +} + /// Convert typed proto policy fields to JSON suitable for `engine.add_data_json()`. /// /// The rego rules reference `data.*` directly, so the JSON structure has @@ -1029,29 +1203,18 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St { allow["fields"] = a.fields.clone().into(); } - let query: serde_json::Map = a - .map(|allow| { - allow - .query - .iter() - .map(|(key, matcher)| { - let mut matcher_json = serde_json::json!({}); - if !matcher.glob.is_empty() { - matcher_json["glob"] = - matcher.glob.clone().into(); - } - if !matcher.any.is_empty() { - matcher_json["any"] = - matcher.any.clone().into(); - } - (key.clone(), matcher_json) - }) - .collect() - }) - .unwrap_or_default(); + let query = a.map_or_else(serde_json::Map::new, |allow| { + l7_matchers_to_json(&allow.query) + }); if !query.is_empty() { allow["query"] = query.into(); } + let params = a.map_or_else(serde_json::Map::new, |allow| { + l7_matchers_to_json(&allow.params) + }); + if !params.is_empty() { + allow["params"] = params.into(); + } serde_json::json!({ "allow": allow }) }) .collect(); @@ -1087,23 +1250,14 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St if !d.fields.is_empty() { deny["fields"] = d.fields.clone().into(); } - let query: serde_json::Map = d - .query - .iter() - .map(|(key, matcher)| { - let mut matcher_json = serde_json::json!({}); - if !matcher.glob.is_empty() { - matcher_json["glob"] = matcher.glob.clone().into(); - } - if !matcher.any.is_empty() { - matcher_json["any"] = matcher.any.clone().into(); - } - (key.clone(), matcher_json) - }) - .collect(); + let query = l7_matchers_to_json(&d.query); if !query.is_empty() { deny["query"] = query.into(); } + let params = l7_matchers_to_json(&d.params); + if !params.is_empty() { + deny["params"] = params.into(); + } deny }) .collect(); @@ -1150,6 +1304,18 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St if e.graphql_max_body_bytes > 0 { ep["graphql_max_body_bytes"] = e.graphql_max_body_bytes.into(); } + if e.json_rpc_max_body_bytes > 0 { + ep["json_rpc_max_body_bytes"] = e.json_rpc_max_body_bytes.into(); + } + if let Some(mcp) = &e.mcp { + if let Some(strict_tool_names) = mcp.strict_tool_names { + ep["mcp_strict_tool_names"] = strict_tool_names.into(); + } + if let Some(allow_all_known_mcp_methods) = mcp.allow_all_known_mcp_methods { + ep["mcp_allow_all_known_mcp_methods"] = + allow_all_known_mcp_methods.into(); + } + } ep }) .collect(); @@ -1957,6 +2123,58 @@ process: }) } + fn l7_jsonrpc_input(host: &str, port: u16, path: &str, method: &str) -> serde_json::Value { + serde_json::json!({ + "network": { "host": host, "port": port }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "POST", + "path": path, + "query_params": {}, + "jsonrpc": { + "method": method + } + } + }) + } + + fn l7_jsonrpc_input_with_params( + host: &str, + port: u16, + path: &str, + method: &str, + params: serde_json::Value, + ) -> serde_json::Value { + let mut input = l7_jsonrpc_input(host, port, path, method); + input["request"]["jsonrpc"]["params"] = params; + input + } + + fn l7_jsonrpc_response_input(host: &str, port: u16, path: &str) -> serde_json::Value { + serde_json::json!({ + "network": { "host": host, "port": port }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "POST", + "path": path, + "query_params": {}, + "jsonrpc": { + "method": null, + "has_response": true, + "error": null + } + } + }) + } + fn l7_graphql_input(host: &str, operations: serde_json::Value) -> serde_json::Value { serde_json::json!({ "network": { "host": host, "port": 443 }, @@ -2024,6 +2242,21 @@ process: val == regorus::Value::from(true) } + fn eval_l7_raw_data(data: serde_json::Value, input: serde_json::Value) -> bool { + let mut engine = regorus::Engine::new(); + engine + .add_policy("policy.rego".into(), TEST_POLICY.into()) + .unwrap(); + engine + .add_data_json(&data.to_string()) + .expect("add raw data json"); + engine.set_input_json(&input.to_string()).unwrap(); + let val = engine + .eval_rule("data.openshell.sandbox.allow_request".into()) + .unwrap(); + val == regorus::Value::from(true) + } + #[test] fn l7_get_allowed_by_rules() { let engine = l7_engine(); @@ -2460,6 +2693,24 @@ network_policies: assert!(eval_l7(&engine, &input)); } + #[test] + fn l7_rest_request_ignores_null_jsonrpc_metadata() { + let engine = l7_engine(); + let mut input = l7_input_with_query( + "api.query.com", + 8080, + "GET", + "/download", + serde_json::json!({ + "tag": ["foo-a"], + }), + ); + input["request"]["graphql"] = serde_json::Value::Null; + input["request"]["jsonrpc"] = serde_json::Value::Null; + + assert!(eval_l7(&engine, &input)); + } + #[test] fn l7_query_missing_required_key_denied() { let engine = l7_engine(); @@ -2503,6 +2754,7 @@ network_policies: operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: std::collections::HashMap::new(), }), }], ..Default::default() @@ -2552,56 +2804,914 @@ network_policies: } #[test] - fn l7_no_request_on_l4_only_endpoint() { - // L4-only endpoint should not match L7 allow_request - let engine = l7_engine(); - let input = l7_input("l4only.example.com", 443, "GET", "/anything"); - assert!(!eval_l7(&engine, &input)); + fn l7_method_from_proto_is_enforced() { + let mut network_policies = std::collections::HashMap::new(); + network_policies.insert( + "jsonrpc_proto".to_string(), + NetworkPolicyRule { + name: "jsonrpc_proto".to_string(), + endpoints: vec![NetworkEndpoint { + host: "jsonrpc.proto.com".to_string(), + port: 8000, + path: "/rpc".to_string(), + protocol: "json-rpc".to_string(), + enforcement: "enforce".to_string(), + rules: vec![L7Rule { + allow: Some(L7Allow { + method: "initialize".to_string(), + path: String::new(), + command: String::new(), + query: std::collections::HashMap::new(), + operation_type: String::new(), + operation_name: String::new(), + fields: Vec::new(), + params: std::collections::HashMap::new(), + }), + }], + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + + let proto = ProtoSandboxPolicy { + version: 1, + filesystem: Some(ProtoFs { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }), + landlock: Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".to_string(), + }), + process: Some(ProtoProc { + run_as_user: "sandbox".to_string(), + run_as_group: "sandbox".to_string(), + }), + network_policies, + }; + + let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); + let allow_input = l7_jsonrpc_input("jsonrpc.proto.com", 8000, "/rpc", "initialize"); + assert!(eval_l7(&engine, &allow_input)); + + let deny_input = l7_jsonrpc_input("jsonrpc.proto.com", 8000, "/rpc", "reports.list"); + assert!(!eval_l7(&engine, &deny_input)); } #[test] - fn l7_wrong_binary_denied_even_with_matching_rules() { - let engine = l7_engine(); - let input = serde_json::json!({ - "network": { "host": "api.example.com", "port": 8080 }, - "exec": { - "path": "/usr/bin/python3", - "ancestors": [], - "cmdline_paths": [] + fn l7_mcp_tool_params_from_proto_are_enforced() { + // Regression: the proto load path (from_proto) must carry the rule + // `params` matcher map. If it is dropped, a tools/call allow rule + // narrowed to one tool degrades to allow-any-tool in production, even + // though the YAML/add_data_json path enforces it correctly. + let mut params = std::collections::HashMap::new(); + params.insert( + "name".to_string(), + L7QueryMatcher { + glob: String::new(), + any: vec!["read_status".to_string(), "submit_*".to_string()], }, - "request": { - "method": "GET", - "path": "/repos/myorg/foo" - } - }); - assert!(!eval_l7(&engine, &input)); - } + ); - #[test] - fn l7_deny_reason_populated() { - let engine = l7_engine(); - let input = l7_input("api.example.com", 8080, "DELETE", "/repos/myorg/foo"); - let mut eng = engine.engine.lock().unwrap(); - eng.set_input_json(&input.to_string()).unwrap(); - let val = eng - .eval_rule("data.openshell.sandbox.request_deny_reason".into()) - .unwrap(); - let reason = match val { - regorus::Value::String(s) => s.to_string(), - _ => String::new(), - }; - assert!( - reason.contains("not permitted"), - "Expected deny reason, got: {reason}" + let mut network_policies = std::collections::HashMap::new(); + network_policies.insert( + "mcp_proto".to_string(), + NetworkPolicyRule { + name: "mcp_proto".to_string(), + endpoints: vec![NetworkEndpoint { + host: "mcp.proto.com".to_string(), + port: 8000, + path: "/mcp".to_string(), + protocol: "mcp".to_string(), + enforcement: "enforce".to_string(), + rules: vec![L7Rule { + allow: Some(L7Allow { + method: "tools/call".to_string(), + path: String::new(), + command: String::new(), + query: std::collections::HashMap::new(), + operation_type: String::new(), + operation_name: String::new(), + fields: Vec::new(), + params, + }), + }], + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, ); - } - #[test] - fn l7_endpoint_config_returned_for_l7_endpoint() { - let engine = l7_engine(); - let input = NetworkInput { - host: "api.example.com".into(), - port: 8080, + let proto = ProtoSandboxPolicy { + version: 1, + filesystem: Some(ProtoFs { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }), + landlock: Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".to_string(), + }), + process: Some(ProtoProc { + run_as_user: "sandbox".to_string(), + run_as_group: "sandbox".to_string(), + }), + network_policies, + }; + + let engine = OpaEngine::from_proto(&proto).expect("engine from proto"); + + let allowed_tool = l7_jsonrpc_input_with_params( + "mcp.proto.com", + 8000, + "/mcp", + "tools/call", + serde_json::json!({ "name": "read_status" }), + ); + assert!( + eval_l7(&engine, &allowed_tool), + "tools/call for an allowed tool should be permitted" + ); + + let blocked_tool = l7_jsonrpc_input_with_params( + "mcp.proto.com", + 8000, + "/mcp", + "tools/call", + serde_json::json!({ "name": "blocked_action" }), + ); + assert!( + !eval_l7(&engine, &blocked_tool), + "tools/call for a non-matching tool must be denied (params matcher must survive the proto load path)" + ); + } + + #[test] + fn l7_jsonrpc_endpoint_ignores_rest_shaped_allow_rules() { + let data = serde_json::json!({ + "network_policies": { + "jsonrpc_rest_bypass": { + "name": "jsonrpc_rest_bypass", + "endpoints": [{ + "host": "jsonrpc.rest-bypass.test", + "ports": [8000], + "path": "/rpc", + "protocol": "json-rpc", + "rules": [{ + "allow": { + "method": "POST", + "path": "**" + } + }] + }], + "binaries": [{ "path": "/usr/bin/curl" }] + } + } + }); + let input = l7_jsonrpc_input("jsonrpc.rest-bypass.test", 8000, "/rpc", "reports.list"); + assert!( + !eval_l7_raw_data(data, input), + "REST-shaped method/path rules must not authorize JSON-RPC endpoints" + ); + } + + #[test] + fn l7_jsonrpc_receive_stream_get_is_denied_for_matching_endpoint() { + let data = r#" +network_policies: + jsonrpc_stream: + name: jsonrpc_stream + endpoints: + - host: jsonrpc.stream.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let receive_stream_get = serde_json::json!({ + "network": { "host": "jsonrpc.stream.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/rpc", + "query_params": {}, + "jsonrpc": { + "method": null, + "receive_stream": true, + "error": null + } + } + }); + assert!(!eval_l7(&engine, &receive_stream_get)); + + let deny_input = serde_json::json!({ + "network": { "host": "jsonrpc.stream.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/other", + "query_params": {}, + "jsonrpc": { + "method": null, + "receive_stream": true, + "error": null + } + } + }); + assert!(!eval_l7(&engine, &deny_input)); + + let bodyless_get_without_receive_stream = serde_json::json!({ + "network": { "host": "jsonrpc.stream.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/rpc", + "query_params": {}, + "jsonrpc": { + "method": null, + "error": null + } + } + }); + assert!(!eval_l7(&engine, &bodyless_get_without_receive_stream)); + + let null_metadata_get = serde_json::json!({ + "network": { "host": "mcp.stream.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/mcp", + "query_params": {}, + "jsonrpc": null + } + }); + assert!(!eval_l7(&engine, &null_metadata_get)); + } + + #[test] + fn l7_mcp_receive_stream_get_is_allowed_for_matching_endpoint() { + let data = r#" +network_policies: + mcp_stream: + name: mcp_stream + endpoints: + - host: mcp.stream.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let allow_input = serde_json::json!({ + "network": { "host": "mcp.stream.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/mcp", + "query_params": {}, + "jsonrpc": { + "method": null, + "params": {}, + "receive_stream": true, + "error": null + } + } + }); + assert!(eval_l7(&engine, &allow_input)); + + let deny_input = serde_json::json!({ + "network": { "host": "mcp.stream.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/other", + "query_params": {}, + "jsonrpc": { + "method": null, + "params": {}, + "receive_stream": true, + "error": null + } + } + }); + assert!(!eval_l7(&engine, &deny_input)); + } + + #[test] + fn l7_jsonrpc_response_post_is_denied_for_matching_endpoint() { + let data = r#" +network_policies: + jsonrpc_response: + name: jsonrpc_response + endpoints: + - host: jsonrpc.response.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let response_input = l7_jsonrpc_response_input("jsonrpc.response.test", 8000, "/rpc"); + assert!(!eval_l7(&engine, &response_input)); + + let mut mixed_input = l7_jsonrpc_input("jsonrpc.response.test", 8000, "/rpc", "initialize"); + mixed_input["request"]["jsonrpc"]["has_response"] = serde_json::json!(true); + assert!(!eval_l7(&engine, &mixed_input)); + + let deny_input = l7_jsonrpc_response_input("jsonrpc.response.test", 8000, "/other"); + assert!(!eval_l7(&engine, &deny_input)); + } + + #[test] + fn l7_mcp_response_post_is_allowed_for_matching_endpoint() { + let data = r#" +network_policies: + mcp_response: + name: mcp_response + endpoints: + - host: mcp.response.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let response_input = l7_jsonrpc_response_input("mcp.response.test", 8000, "/mcp"); + assert!(eval_l7(&engine, &response_input)); + + let deny_input = l7_jsonrpc_response_input("mcp.response.test", 8000, "/other"); + assert!(!eval_l7(&engine, &deny_input)); + } + + #[test] + fn l7_jsonrpc_unlisted_method_is_denied() { + let data = r#" +network_policies: + jsonrpc_methods: + name: jsonrpc_methods + endpoints: + - host: jsonrpc.methods.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let unlisted_input = + l7_jsonrpc_input("jsonrpc.methods.test", 8000, "/rpc", "reports.progress"); + + assert!(!eval_l7(&engine, &unlisted_input)); + } + + #[test] + fn l7_method_rules_require_post() { + let data = r#" +network_policies: + jsonrpc_post: + name: jsonrpc_post + endpoints: + - host: jsonrpc.post.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: initialize + deny_rules: + - method: reports.archive + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + + let mut post_input = l7_jsonrpc_input("jsonrpc.post.test", 8000, "/rpc", "initialize"); + assert!(eval_l7(&engine, &post_input)); + + post_input["request"]["method"] = serde_json::json!("PUT"); + assert!(!eval_l7(&engine, &post_input)); + + let mut get_with_method = l7_jsonrpc_input("jsonrpc.post.test", 8000, "/rpc", "initialize"); + get_with_method["request"]["method"] = serde_json::json!("GET"); + assert!(!eval_l7(&engine, &get_with_method)); + } + + #[test] + fn l7_jsonrpc_request_params_do_not_affect_method_policy() { + let data = r#" +network_policies: + jsonrpc_params: + name: jsonrpc_params + endpoints: + - host: jsonrpc.params.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: reports.search + deny_rules: + - method: reports.archive + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + + let read_status = l7_jsonrpc_input_with_params( + "jsonrpc.params.test", + 8000, + "/rpc", + "reports.search", + serde_json::json!({"query": "quarterly"}), + ); + assert!(eval_l7(&engine, &read_status)); + + let submit_report = l7_jsonrpc_input_with_params( + "jsonrpc.params.test", + 8000, + "/rpc", + "reports.search", + serde_json::json!({ + "query": "quarterly", + "filters.scope": "workspace/main" + }), + ); + assert!(eval_l7(&engine, &submit_report)); + + let blocked_without_args = l7_jsonrpc_input_with_params( + "jsonrpc.params.test", + 8000, + "/rpc", + "reports.search", + serde_json::json!({"query": "blocked"}), + ); + assert!(eval_l7(&engine, &blocked_without_args)); + + let blocked_with_args = l7_jsonrpc_input_with_params( + "jsonrpc.params.test", + 8000, + "/rpc", + "reports.search", + serde_json::json!({ + "query": "blocked", + "filters.reason": "test" + }), + ); + assert!(eval_l7(&engine, &blocked_with_args)); + } + + #[test] + fn l7_jsonrpc_method_globs_are_exact_literals_in_rego() { + let data = serde_json::json!({ + "network_policies": { + "jsonrpc_glob_literal": { + "name": "jsonrpc_glob_literal", + "endpoints": [{ + "host": "jsonrpc.glob-literal.test", + "ports": [8000], + "path": "/rpc", + "protocol": "json-rpc", + "rules": [{ + "allow": { + "method": "reports.*" + } + }] + }], + "binaries": [{ "path": "/usr/bin/curl" }] + } + } + }); + + let glob_match_candidate = + l7_jsonrpc_input("jsonrpc.glob-literal.test", 8000, "/rpc", "reports.list"); + assert!( + !eval_l7_raw_data(data.clone(), glob_match_candidate), + "generic JSON-RPC method rules must not use glob semantics" + ); + + let exact_literal = + l7_jsonrpc_input("jsonrpc.glob-literal.test", 8000, "/rpc", "reports.*"); + assert!( + eval_l7_raw_data(data, exact_literal), + "generic JSON-RPC method rules should use exact method equality" + ); + } + + #[test] + fn l7_jsonrpc_allow_all_still_allows_any_method() { + let data = r#" +network_policies: + jsonrpc_allow_all: + name: jsonrpc_allow_all + endpoints: + - host: jsonrpc.allow-all.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: "*" + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + + let initialize = l7_jsonrpc_input("jsonrpc.allow-all.test", 8000, "/rpc", "initialize"); + assert!(eval_l7(&engine, &initialize)); + + let archive_report = + l7_jsonrpc_input("jsonrpc.allow-all.test", 8000, "/rpc", "reports.archive"); + assert!(eval_l7(&engine, &archive_report)); + } + + #[test] + fn l7_mcp_rules_filter_tools_call() { + let data = r#" +network_policies: + mcp_params: + name: mcp_params + endpoints: + - host: mcp.params.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + max_body_bytes: 131072 + rules: + - allow: + method: tools/call + tool: + any: [read_status, submit_*] + deny_rules: + - method: tools/call + tool: blocked_action + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + + let read_status = l7_jsonrpc_input_with_params( + "mcp.params.test", + 8000, + "/mcp", + "tools/call", + serde_json::json!({ + "name": "read_status", + "arguments.scope": "workspace/main" + }), + ); + assert!(eval_l7(&engine, &read_status)); + + let read_status_any_args = l7_jsonrpc_input_with_params( + "mcp.params.test", + 8000, + "/mcp", + "tools/call", + serde_json::json!({ + "name": "read_status", + "arguments.scope": "workspace/other" + }), + ); + assert!(eval_l7(&engine, &read_status_any_args)); + + let submit_report = l7_jsonrpc_input_with_params( + "mcp.params.test", + 8000, + "/mcp", + "tools/call", + serde_json::json!({"name": "submit_report"}), + ); + assert!(eval_l7(&engine, &submit_report)); + + let blocked = l7_jsonrpc_input_with_params( + "mcp.params.test", + 8000, + "/mcp", + "tools/call", + serde_json::json!({"name": "blocked_action"}), + ); + assert!(!eval_l7(&engine, &blocked)); + + let list_tools = l7_jsonrpc_input("mcp.params.test", 8000, "/mcp", "tools/list"); + assert!(!eval_l7(&engine, &list_tools)); + } + + #[test] + fn l7_mcp_method_profile_allows_all_tools() { + let data = r#" +network_policies: + mcp_default: + name: mcp_default + endpoints: + - host: mcp.default.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + allow_all_known_mcp_methods: true + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + + let tool_call = l7_jsonrpc_input_with_params( + "mcp.default.test", + 8000, + "/mcp", + "tools/call", + serde_json::json!({ + "name": "any_tool", + "arguments.scope": "workspace/other" + }), + ); + assert!(eval_l7(&engine, &tool_call)); + + let list_tools = l7_jsonrpc_input("mcp.default.test", 8000, "/mcp", "tools/list"); + assert!(eval_l7(&engine, &list_tools)); + } + + #[test] + fn l7_jsonrpc_null_metadata_non_matches_without_opa_error() { + let data = r#" +network_policies: + jsonrpc_null: + name: jsonrpc_null + endpoints: + - host: jsonrpc.null.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: reports.list + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let input = serde_json::json!({ + "network": { "host": "jsonrpc.null.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "POST", + "path": "/rpc", + "query_params": {}, + "jsonrpc": null + } + }); + + assert!(!eval_l7(&engine, &input)); + } + + #[test] + fn l7_mcp_null_params_non_matches_without_opa_error() { + let data = r#" +network_policies: + mcp_null_params: + name: mcp_null_params + endpoints: + - host: mcp.null-params.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + rules: + - allow: + method: tools/call + tool: read_status + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let input = serde_json::json!({ + "network": { "host": "mcp.null-params.test", "port": 8000 }, + "exec": { + "path": "/usr/bin/curl", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "POST", + "path": "/mcp", + "query_params": {}, + "jsonrpc": { + "method": "tools/call", + "params": null + } + } + }); + + assert!(!eval_l7(&engine, &input)); + } + + #[test] + fn l7_jsonrpc_params_matchers_are_rejected() { + let data = r#" +network_policies: + invalid_jsonrpc_params: + name: invalid_jsonrpc_params + endpoints: + - host: jsonrpc.invalid.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: reports.search + params: + query: quarterly + binaries: + - { path: /usr/bin/curl } +"#; + let Err(err) = OpaEngine::from_strings(TEST_POLICY, data) else { + panic!("JSON-RPC params matchers should fail validation"); + }; + + assert!( + err.to_string().contains("do not support params"), + "unexpected validation error: {err}" + ); + } + + #[test] + fn l7_jsonrpc_config_alias_unknown_fields_are_rejected() { + let data = r#" +network_policies: + invalid_jsonrpc_config: + name: invalid_jsonrpc_config + endpoints: + - host: jsonrpc.invalid-config.test + port: 8000 + path: /rpc + protocol: json-rpc + enforcement: enforce + json_rpc: + on_parse_error: allow + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let Err(err) = OpaEngine::from_strings(TEST_POLICY, data) else { + panic!("unknown JSON-RPC config fields should fail validation"); + }; + + let message = err.to_string(); + assert!( + message.contains("json_rpc") && message.contains("on_parse_error"), + "unexpected validation error: {err}" + ); + } + + #[test] + fn l7_mcp_config_alias_types_are_rejected() { + let data = r#" +network_policies: + invalid_mcp_config: + name: invalid_mcp_config + endpoints: + - host: mcp.invalid-config.test + port: 8000 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + max_body_bytes: large + rules: + - allow: + method: initialize + binaries: + - { path: /usr/bin/curl } +"#; + let Err(err) = OpaEngine::from_strings(TEST_POLICY, data) else { + panic!("mistyped MCP config fields should fail validation"); + }; + + let message = err.to_string(); + assert!( + message.contains("mcp") && message.contains("large"), + "unexpected validation error: {err}" + ); + } + + #[test] + fn l7_no_request_on_l4_only_endpoint() { + // L4-only endpoint should not match L7 allow_request + let engine = l7_engine(); + let input = l7_input("l4only.example.com", 443, "GET", "/anything"); + assert!(!eval_l7(&engine, &input)); + } + + #[test] + fn l7_wrong_binary_denied_even_with_matching_rules() { + let engine = l7_engine(); + let input = serde_json::json!({ + "network": { "host": "api.example.com", "port": 8080 }, + "exec": { + "path": "/usr/bin/python3", + "ancestors": [], + "cmdline_paths": [] + }, + "request": { + "method": "GET", + "path": "/repos/myorg/foo" + } + }); + assert!(!eval_l7(&engine, &input)); + } + + #[test] + fn l7_deny_reason_populated() { + let engine = l7_engine(); + let input = l7_input("api.example.com", 8080, "DELETE", "/repos/myorg/foo"); + let mut eng = engine.engine.lock().unwrap(); + eng.set_input_json(&input.to_string()).unwrap(); + let val = eng + .eval_rule("data.openshell.sandbox.request_deny_reason".into()) + .unwrap(); + let reason = match val { + regorus::Value::String(s) => s.to_string(), + _ => String::new(), + }; + assert!( + reason.contains("not permitted"), + "Expected deny reason, got: {reason}" + ); + } + + #[test] + fn l7_endpoint_config_returned_for_l7_endpoint() { + let engine = l7_engine(); + let input = NetworkInput { + host: "api.example.com".into(), + port: 8080, binary_path: PathBuf::from("/usr/bin/curl"), binary_sha256: "unused".into(), ancestors: vec![], @@ -2615,6 +3725,44 @@ network_policies: assert_eq!(l7.enforcement, crate::l7::EnforcementMode::Enforce); } + #[test] + fn l7_endpoint_config_preserves_mcp_strict_tool_names_opt_out() { + let data = r#" +network_policies: + mcp: + name: mcp + endpoints: + - host: mcp.example.com + port: 443 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + strict_tool_names: false + rules: + - allow: + method: tools/call + binaries: + - { path: /usr/bin/curl } +"#; + let engine = OpaEngine::from_strings(TEST_POLICY, data).expect("engine from yaml"); + let input = NetworkInput { + host: "mcp.example.com".into(), + port: 443, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let config = engine + .query_endpoint_config(&input) + .expect("query endpoint config") + .expect("expected mcp endpoint config"); + let l7 = crate::l7::parse_l7_config(&config).expect("parse l7 config"); + assert_eq!(l7.protocol, crate::l7::L7Protocol::Mcp); + assert!(!l7.mcp_strict_tool_names); + } + #[test] fn l7_endpoint_config_preserves_proto_allow_encoded_slash() { let mut network_policies = std::collections::HashMap::new(); diff --git a/crates/openshell-supervisor-network/src/policy_local.rs b/crates/openshell-supervisor-network/src/policy_local.rs index b3b6842479..3cbc315026 100644 --- a/crates/openshell-supervisor-network/src/policy_local.rs +++ b/crates/openshell-supervisor-network/src/policy_local.rs @@ -1088,6 +1088,7 @@ fn network_endpoint_from_json( operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::new(), }), }) .collect(); @@ -1102,6 +1103,7 @@ fn network_endpoint_from_json( operation_type: String::new(), operation_name: String::new(), fields: Vec::new(), + params: HashMap::new(), }) .collect(); @@ -1125,6 +1127,8 @@ fn network_endpoint_from_json( persisted_queries: String::new(), graphql_persisted_queries: HashMap::new(), graphql_max_body_bytes: 0, + json_rpc_max_body_bytes: 0, + mcp: None, path: String::new(), credential_signing: String::new(), signing_service: String::new(), diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index bba3c3919a..0d2c8c0258 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -13,7 +13,7 @@ use openshell_core::denial::DenialEvent; use openshell_core::net::{is_always_blocked_ip, is_internal_ip, is_link_local_ip}; use openshell_core::policy::ProxyPolicy; use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_core::secrets::{SecretResolver, rewrite_header_line_checked}; +use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, @@ -185,7 +185,7 @@ impl ProxyHandle { /// The proxy uses OPA for network decisions with process-identity binding /// via `/proc/net/tcp`. All connections are evaluated through OPA policy. #[allow(clippy::too_many_arguments)] - pub async fn start_with_bind_addr( + pub(crate) async fn start_with_bind_addr( policy: &ProxyPolicy, bind_addr: Option, opa_engine: Arc, @@ -454,6 +454,27 @@ fn emit_forward_success_activity(tx: Option<&ActivitySender>, l7_activity_pendin ); } +fn forward_l7_hard_deny_reason( + protocol: crate::l7::L7Protocol, + request_info: &crate::l7::L7RequestInfo, +) -> Option { + request_info + .graphql + .as_ref() + .and_then(|info| info.error.as_deref()) + .map(|error| format!("GraphQL request rejected: {error}")) + .or_else(|| { + request_info.jsonrpc.as_ref().and_then(|info| { + info.error + .as_deref() + .map(|error| format!("JSON-RPC request rejected: {error}")) + .or_else(|| { + crate::l7::relay::jsonrpc_response_frame_hard_deny_reason(protocol, info) + }) + }) + }) +} + /// Emit a denial event to the aggregator channel (if configured). /// Used by `handle_tcp_connection` which owns `Option`. fn emit_denial( @@ -601,6 +622,7 @@ async fn handle_tcp_connection( ) .await?; if let InferenceOutcome::Denied { reason } = outcome { + emit_activity(&activity_tx, true, "forward_policy"); let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) @@ -2918,16 +2940,14 @@ fn rewrite_forward_request( path: &str, secret_resolver: Option<&SecretResolver>, request_body_credential_rewrite: bool, -) -> Result, openshell_core::secrets::UnresolvedPlaceholderError> { +) -> Result, secrets::UnresolvedPlaceholderError> { let header_end = raw[..used] .windows(4) .position(|w| w == b"\r\n\r\n") .map_or(used, |p| p + 4); let websocket_upgrade = crate::l7::rest::request_is_websocket_upgrade(&raw[..header_end]); let upstream_path = match secret_resolver { - Some(resolver) => { - openshell_core::secrets::rewrite_target_for_eval(path, resolver)?.resolved - } + Some(resolver) => secrets::rewrite_target_for_eval(path, resolver)?.resolved, None => path.to_string(), }; @@ -3020,10 +3040,10 @@ fn rewrite_forward_request( output.len() }; let output_str = String::from_utf8_lossy(&output[..scan_end]); - if output_str.contains(openshell_core::secrets::PLACEHOLDER_PREFIX_PUBLIC) - || output_str.contains(openshell_core::secrets::PROVIDER_ALIAS_MARKER_PUBLIC) + if output_str.contains(secrets::PLACEHOLDER_PREFIX_PUBLIC) + || output_str.contains(secrets::PROVIDER_ALIAS_MARKER_PUBLIC) { - return Err(openshell_core::secrets::UnresolvedPlaceholderError { location: "header" }); + return Err(secrets::UnresolvedPlaceholderError { location: "header" }); } } @@ -3597,18 +3617,75 @@ async fn handle_forward_proxy( } else { None }; + let jsonrpc = if l7_config.config.protocol.is_jsonrpc_family() { + let header_end = forward_request_bytes + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(forward_request_bytes.len(), |p| p + 4); + let header_str = std::str::from_utf8(&forward_request_bytes[..header_end]) + .map_err(|_| miette::miette!("Forward JSON-RPC headers contain invalid UTF-8"))?; + let body_length = crate::l7::rest::parse_body_length(header_str)?; + let mut jsonrpc_request = crate::l7::provider::L7Request { + action: method.to_string(), + target: path.clone(), + query_params: query_params.clone(), + raw_header: forward_request_bytes, + body_length, + }; + if crate::l7::jsonrpc::jsonrpc_receive_stream_request(&jsonrpc_request) { + forward_request_bytes = jsonrpc_request.raw_header; + Some(crate::l7::jsonrpc::JsonRpcRequestInfo::receive_stream()) + } else { + let body = match crate::l7::http::read_body_for_inspection( + client, + &mut jsonrpc_request, + l7_config.config.json_rpc_max_body_bytes, + ) + .await + { + Ok(body) => body, + Err(e) => { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .message(format!("FORWARD_JSONRPC_L7 request rejected: {e}")) + .build(); + ocsf_emit!(event); + emit_activity_simple(activity_tx, true, "l7_parse_rejection"); + respond( + client, + &build_json_error_response( + 400, + "Bad Request", + "invalid_jsonrpc_request", + &format!("JSON-RPC request rejected before policy evaluation: {e}"), + ), + ) + .await?; + return Ok(()); + } + }; + forward_request_bytes = jsonrpc_request.raw_header; + Some(crate::l7::jsonrpc::parse_jsonrpc_body_with_options( + &body, + crate::l7::jsonrpc::JsonRpcInspectionOptions::for_config(&l7_config.config), + )) + } + } else { + None + }; let request_info = crate::l7::L7RequestInfo { action: method.to_string(), target: path.clone(), query_params, graphql, + jsonrpc, }; - let parse_error_reason = request_info - .graphql - .as_ref() - .and_then(|info| info.error.as_deref()) - .map(|error| format!("GraphQL request rejected: {error}")); + let parse_error_reason = + forward_l7_hard_deny_reason(l7_config.config.protocol, &request_info); let force_deny = parse_error_reason.is_some(); let (allowed, reason) = parse_error_reason.map_or_else( || { @@ -3649,16 +3726,36 @@ async fn handle_forward_proxy( SeverityId::Informational, ), }; - let engine_type = if l7_config.config.protocol == crate::l7::L7Protocol::Graphql { - "l7-graphql" - } else { - "l7" - }; - let message_prefix = if l7_config.config.protocol == crate::l7::L7Protocol::Graphql { - "FORWARD_GRAPHQL_L7" - } else { - "FORWARD_L7" + let engine_type = match l7_config.config.protocol { + crate::l7::L7Protocol::Graphql => "l7-graphql", + crate::l7::L7Protocol::JsonRpc => "l7-jsonrpc", + crate::l7::L7Protocol::Mcp => "l7-mcp", + _ => "l7", }; + let log_message = request_info.jsonrpc.as_ref().map_or_else( + || { + let message_prefix = + if l7_config.config.protocol == crate::l7::L7Protocol::Graphql { + "FORWARD_GRAPHQL_L7" + } else { + "FORWARD_L7" + }; + format!( + "{message_prefix} {decision_str} {method} {host_lc}:{port}{path} reason={reason}" + ) + }, + |jsonrpc_info| { + let endpoint = format!("{host_lc}:{port}{path}"); + crate::l7::relay::jsonrpc_log_message( + decision_str, + method, + &endpoint, + jsonrpc_info, + tunnel_engine.captured_generation(), + &reason, + ) + }, + ); let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Other) .action(action_id) @@ -3675,9 +3772,7 @@ async fn handle_forward_proxy( .with_cmd_line(&cmdline_str), ) .firewall_rule(policy_str, engine_type) - .message(format!( - "{message_prefix} {decision_str} {method} {host_lc}:{port}{path} reason={reason}" - )) + .message(log_message) .build(); ocsf_emit!(event); } @@ -4294,6 +4389,8 @@ mod tests { tls: crate::l7::TlsMode::Auto, enforcement: crate::l7::EnforcementMode::Enforce, graphql_max_body_bytes: crate::l7::graphql::DEFAULT_MAX_BODY_BYTES, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, allow_encoded_slash: false, websocket_credential_rewrite, request_body_credential_rewrite: false, @@ -4455,6 +4552,57 @@ network_policies: assert_eq!(event.deny_group, "unknown"); } + #[test] + fn forward_l7_hard_deny_reason_includes_jsonrpc_errors() { + let request_info = crate::l7::L7RequestInfo { + action: "POST".to_string(), + target: "/rpc".to_string(), + query_params: std::collections::HashMap::new(), + graphql: None, + jsonrpc: Some(crate::l7::jsonrpc::JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: false, + receive_stream: false, + has_response: false, + error: Some("missing or non-string 'jsonrpc' field".to_string()), + }), + }; + + let reason = forward_l7_hard_deny_reason(crate::l7::L7Protocol::JsonRpc, &request_info) + .expect("JSON-RPC parse error"); + + assert_eq!( + reason, + "JSON-RPC request rejected: missing or non-string 'jsonrpc' field" + ); + } + + #[test] + fn forward_l7_hard_deny_reason_includes_jsonrpc_response_frames() { + let request_info = crate::l7::L7RequestInfo { + action: "POST".to_string(), + target: "/rpc".to_string(), + query_params: std::collections::HashMap::new(), + graphql: None, + jsonrpc: Some(crate::l7::jsonrpc::JsonRpcRequestInfo { + calls: Vec::new(), + is_batch: false, + receive_stream: false, + has_response: true, + error: None, + }), + }; + + let reason = forward_l7_hard_deny_reason(crate::l7::L7Protocol::JsonRpc, &request_info) + .expect("JSON-RPC response hard deny"); + + assert_eq!(reason, crate::l7::relay::JSONRPC_RESPONSE_FRAME_DENY_REASON); + assert!( + forward_l7_hard_deny_reason(crate::l7::L7Protocol::Mcp, &request_info).is_none(), + "MCP response frames are evaluated by policy instead of hard-denied" + ); + } + #[test] fn forward_l7_allowed_activity_is_deferred_until_after_ssrf() { let (tx, mut rx) = mpsc::channel(4); @@ -5013,6 +5161,8 @@ network_policies: tls: crate::l7::TlsMode::Auto, enforcement: crate::l7::EnforcementMode::Enforce, graphql_max_body_bytes: crate::l7::graphql::DEFAULT_MAX_BODY_BYTES, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, allow_encoded_slash: false, websocket_credential_rewrite: false, request_body_credential_rewrite: false, @@ -5029,6 +5179,8 @@ network_policies: tls: crate::l7::TlsMode::Auto, enforcement: crate::l7::EnforcementMode::Enforce, graphql_max_body_bytes: crate::l7::graphql::DEFAULT_MAX_BODY_BYTES, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, allow_encoded_slash: false, websocket_credential_rewrite: false, request_body_credential_rewrite: false, diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 5dc8d9a5ed..906d55d58d 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -155,11 +155,11 @@ Each endpoint defines a reachable destination and optional inspection rules. | `host` | string | Yes | Hostname or IP address. Supports a `*` wildcard inside the first DNS label only: `*.example.com`, `**.example.com`, and intra-label patterns like `*-aiplatform.googleapis.com` are accepted; bare `*`/`**`, TLD wildcards (`*.com`), and wildcards outside the first label are rejected at load time. | | `port` | integer | Yes | TCP port number. | | `path` | string | No | Optional HTTP path glob used to select between L7 endpoints that share the same host and port. Empty means all paths. Use this when REST and GraphQL live under the same host, such as `/repos/**` and `/graphql`. | -| `protocol` | string | No | Set to `rest` for HTTP method/path inspection, `websocket` for RFC 6455 upgrade and client text-message inspection, or `graphql` for GraphQL-over-HTTP operation inspection. WebSocket endpoints can also use GraphQL operation rules for GraphQL-over-WebSocket traffic. Omit for TCP passthrough. | +| `protocol` | string | No | Set to `rest` for HTTP method/path inspection, `websocket` for RFC 6455 upgrade and client text-message inspection, `graphql` for GraphQL-over-HTTP operation inspection, `mcp` for MCP Streamable HTTP request inspection, or `json-rpc` for generic JSON-RPC-over-HTTP method inspection. WebSocket endpoints can also use GraphQL operation rules for GraphQL-over-WebSocket traffic. Omit for TCP passthrough. | | `tls` | string | No | TLS handling mode. The proxy auto-detects TLS by peeking the first bytes of each connection and terminates it for inspected HTTPS traffic, so this field is optional in most cases. Set to `skip` to disable auto-detection for edge cases such as client-certificate mTLS or non-standard protocols. The values `terminate` and `passthrough` are deprecated and log a warning; they are still accepted for backward compatibility but have no effect on behavior. | | `enforcement` | string | No | `enforce` actively blocks disallowed requests. `audit` logs violations but allows traffic through. | -| `access` | string | No | Access preset. One of `read-only`, `read-write`, or `full`. Mutually exclusive with `rules`. | -| `rules` | list of rule objects | No | Fine-grained protocol-specific allow rules. Mutually exclusive with `access`. | +| `access` | string | No | Access preset. One of `read-only`, `read-write`, or `full`. Mutually exclusive with `rules`. Not valid on `protocol: mcp` or `protocol: json-rpc`; MCP uses explicit rules unless `mcp.allow_all_known_mcp_methods: true` enables the endpoint method profile, and JSON-RPC always uses explicit rules. | +| `rules` | list of allow rule objects | No | Fine-grained protocol-specific allow rules. Mutually exclusive with `access`. | | `deny_rules` | list of deny rule objects | No | L7 deny rules that block specific requests even when allowed by `access` or `rules`. Deny rules take precedence over allow rules. | | `allowed_ips` | list of string | No | CIDR or IP allowlist for SSRF override. Exact user-declared hostname endpoints may resolve to RFC 1918 private addresses without this field, but wildcard, hostless, and policy-advisor-proposed endpoints still require `allowed_ips` for private resolved IPs. Entries overlapping loopback (`127.0.0.0/8`), link-local (`169.254.0.0/16`), or unspecified (`0.0.0.0`) are rejected at load time. | | `allow_encoded_slash` | bool | No | When `true`, L7 request parsing preserves `%2F` inside path segments instead of rejecting it. Use this for registries and APIs such as npm scoped packages (`/@scope%2Fname`). Defaults to `false`. | @@ -171,18 +171,25 @@ Each endpoint defines a reachable destination and optional inspection rules. | `persisted_queries` | string | No | GraphQL hash-only behavior for `protocol: graphql` and GraphQL-over-WebSocket operation policy. Default is `deny`; use `allow_registered` only with `graphql_persisted_queries`. | | `graphql_persisted_queries` | map | No | Trusted GraphQL persisted-query registry keyed by hash or saved-query ID. Values contain `operation_type`, optional `operation_name`, and optional root `fields`. | | `graphql_max_body_bytes` | integer | No | Maximum GraphQL-over-HTTP request body bytes buffered for inspection. Defaults to `65536`. | +| `mcp` | object | No | MCP endpoint options for `protocol: mcp`. MCP endpoints must set a concrete `host` and `port` or `ports`; `protocol: mcp` alone is invalid and is not treated as a wildcard endpoint. | +| `mcp.max_body_bytes` | integer | No | Maximum MCP JSON-RPC-over-HTTP request body bytes buffered for inspection. Defaults to `65536`. | +| `mcp.strict_tool_names` | bool | No | Defaults to `true`. Requires `tools/call` `params.name` values to match `^[A-Za-z0-9_.-]{1,128}$` before policy evaluation. Set to `false` only for compatibility with MCP servers that intentionally use non-recommended tool names. Wildcard `tool` matchers require this to remain enabled. | +| `mcp.allow_all_known_mcp_methods` | bool | No | Defaults to `false`. When `true`, enables the endpoint MCP method profile: omitted `rules` allow all MCP-family methods and all tools before `deny_rules`, and omitted rule `method` uses that profile. When unset or `false`, explicit MCP method rules are required; rules with `tool` or `params.name` must set `method: tools/call`. | +| `json_rpc` | object | No | JSON-RPC endpoint options. For `protocol: json-rpc`, `json_rpc.max_body_bytes` sets the maximum JSON-RPC-over-HTTP request body bytes buffered for inspection. Defaults to `65536`. | Credential rewrite recognizes the canonical `openshell:resolve:env:KEY` placeholder form and whole-token provider-shaped aliases such as `provider-OPENSHELL-RESOLVE-ENV-API_TOKEN` when the referenced environment key exists in the configured provider credentials. #### Access Levels -The `access` field accepts one of the following values: +The `access` field accepts one of the following values on REST, WebSocket, and GraphQL endpoints. MCP and JSON-RPC endpoints reject `access` because HTTP method/path presets cannot authorize JSON-RPC safely. Use explicit MCP rules, set `mcp.allow_all_known_mcp_methods: true` for the MCP method profile, or use explicit JSON-RPC rules. -| Value | REST expansion | WebSocket expansion | GraphQL expansion | -|---|---|---|---| -| `full` | All methods and paths. | WebSocket upgrade and all inspected client text-message paths. | All operation types. | -| `read-only` | `GET`, `HEAD`, `OPTIONS`. | WebSocket upgrade handshake only. | `query` operations. | -| `read-write` | `GET`, `HEAD`, `OPTIONS`, `POST`, `PUT`, `PATCH`. | WebSocket upgrade handshake and client text messages. | `query` and `mutation` operations. | +| Value | REST expansion | WebSocket expansion | GraphQL expansion | MCP / JSON-RPC expansion | +|---|---|---|---|---| +| `full` | All methods and paths. | WebSocket upgrade and all inspected client text-message paths. | All operation types. | Rejected. | +| `read-only` | `GET`, `HEAD`, `OPTIONS`. | WebSocket upgrade handshake only. | `query` operations. | Rejected. | +| `read-write` | `GET`, `HEAD`, `OPTIONS`, `POST`, `PUT`, `PATCH`. | WebSocket upgrade handshake and client text messages. | `query` and `mutation` operations. | Rejected. | + +For MCP endpoints, configure explicit `rules` with `method`, optional `tool`, and supported `params`. For generic JSON-RPC endpoints, configure explicit `rules` with `method`; JSON-RPC policy `params` matchers are not presently supported. #### Allow Rule Objects @@ -277,6 +284,74 @@ rules: Do not combine `method`, `path`, or `query` with `operation_type`, `operation_name`, or `fields` inside the same WebSocket rule. When a WebSocket endpoint has GraphQL operation policy, use GraphQL rules for client messages instead of a raw `WEBSOCKET_TEXT` allow rule. +##### MCP Allow And Deny Rules (`protocol: mcp`) + +MCP rules match sandbox-to-server MCP Streamable HTTP request bodies by MCP method and optional tool selectors. OpenShell parses the underlying JSON-RPC 2.0 envelope, validates known MCP request and notification params, and preserves unknown extension methods as policy-addressable literal method strings. Until OpenShell exposes explicit MCP version profiles, `mcp.allow_all_known_mcp_methods` defaults to `false`, so endpoints require explicit MCP method rules. Set `mcp.allow_all_known_mcp_methods: true` to enable the endpoint method profile; in that mode, rules can omit `method`, and tool selectors are normalized to `tools/call` internally. By default, `tools/call` `params.name` must match the MCP-recommended tool-name pattern `^[A-Za-z0-9_.-]{1,128}$`; configure `mcp.strict_tool_names: false` on the endpoint only to allow a server that intentionally uses names outside that pattern. Wildcard `tool` matchers require `mcp.strict_tool_names` to remain enabled. JSON-RPC responses and server-to-client MCP messages on response bodies or SSE streams are relayed but are not currently parsed for policy enforcement. + +Use `rules` for MCP allow rules and `deny_rules` for MCP deny rules. Deny rules take precedence over allow rules. If an MCP endpoint sets `mcp.allow_all_known_mcp_methods: true` and omits `rules`, OpenShell allows all MCP-family methods and all tools, then applies any `deny_rules`. Otherwise, the endpoint must define explicit rules. A broad allow or deny rule whose method matcher includes `tools/call` cannot be combined with tool-specific allow rules because it would bypass or erase the tool filter; add `tool` or `params.name` to scope `tools/call`, or remove the tool-specific rules. In a batch request, one denied call denies the full batch. + +| Field | Type | Required | Description | +|---|---|---|---| +| `method` | string | No | MCP method name, such as `initialize`, `tools/list`, `tools/call`, or an unknown extension method. Globs are accepted only for the `tools/` method family, such as `tools/*`. Required unless `mcp.allow_all_known_mcp_methods` is `true`; when that option is true, omitted method uses the endpoint method profile. Do not use `method: "*"` for MCP; omit `method` only when using the allow-all MCP method profile. | +| `tool` | string or matcher | No | Convenience matcher for `tools/call` `params.name`. Supports a glob string or `{ any: [...] }`. Requires `method: tools/call` unless `mcp.allow_all_known_mcp_methods` is `true`; validation fails otherwise. Omit to match every tool. | +| `params` | map | No | MCP currently accepts only `params.name` as a lower-level tool-name matcher. Requires `method: tools/call` unless `mcp.allow_all_known_mcp_methods` is `true`; validation fails otherwise. Tool argument matching is not supported yet; allowed tools accept all argument payloads by default. | + +Example MCP rules: + +```yaml showLineNumbers={false} +endpoints: + - host: mcp.example.com + port: 443 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + max_body_bytes: 131072 + rules: + - allow: + method: tools/call + tool: search_web + - allow: + method: tools/call + tool: + any: [create_issue, list_issues] + deny_rules: + - method: tools/call + tool: send_email + - method: tools/call + tool: execute_code +``` + +##### JSON-RPC Allow Rule (`protocol: json-rpc`) + +JSON-RPC allow rules match sandbox-to-server JSON-RPC-over-HTTP request objects by RPC method. They apply to single JSON-RPC requests and batch requests. For a batch, OpenShell evaluates each call independently. Client-to-server JSON-RPC response frames in POST bodies are denied. Server-to-client messages on HTTP response bodies or MCP SSE streams are relayed but are not currently parsed for policy enforcement. + +| Field | Type | Required | Description | +|---|---|---|---| +| `method` | string | Yes | Exact JSON-RPC method name such as `initialize` or `reports.search`. Use `*` only as the allow-all sentinel. Other wildcard or glob patterns are rejected for generic JSON-RPC endpoints. | + +Generic JSON-RPC policy `params` matchers are not supported. Allow rules match only the JSON-RPC method. + +Example JSON-RPC allow rules: + +```yaml showLineNumbers={false} +endpoints: + - host: jsonrpc.example.com + port: 443 + path: /rpc + protocol: json-rpc + enforcement: enforce + json_rpc: + max_body_bytes: 131072 + rules: + - allow: + method: initialize + - allow: + method: reports.list + - allow: + method: reports.search +``` + #### Deny Rule Objects Blocks specific operations on endpoints that otherwise have broad access. Deny rules are evaluated after allow rules and take precedence: if a request matches any deny rule, it is blocked regardless of what the allow rules or access preset permit. @@ -359,6 +434,32 @@ endpoints: operation_name: Admin* ``` +##### JSON-RPC Deny Rule (`protocol: json-rpc`) + +JSON-RPC deny rules use the same field names as JSON-RPC allow rules, but they appear directly under each `deny_rules` entry instead of under an `allow` wrapper. Deny rules take precedence over allow rules. In a batch request, one denied call denies the full batch. + +| Field | Type | Required | Description | +|---|---|---|---| +| `method` | string | Yes | Exact JSON-RPC method name to deny, or `*` to deny all JSON-RPC methods. Other wildcard or glob patterns are rejected for generic JSON-RPC endpoints. | + +JSON-RPC deny rules do not support policy `params` matchers yet. Use MCP `tool` rules for MCP tool calls, or deny generic JSON-RPC methods by `method`. + +Example JSON-RPC deny rules: + +```yaml showLineNumbers={false} +endpoints: + - host: jsonrpc.example.com + port: 443 + path: /rpc + protocol: json-rpc + enforcement: enforce + rules: + - allow: + method: "*" + deny_rules: + - method: reports.delete +``` + ### Binary Object Identifies an executable that is permitted to use the associated endpoints. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 434ff7c6e0..ea87164225 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -148,7 +148,7 @@ The following steps outline the hot-reload policy update workflow. To inspect the effective policy that the sandbox enforces, including provider-composed entries, use `openshell policy get --full`. To inspect a stored sandbox-authored revision instead of the current effective policy, pass `--rev `. -5. Edit the YAML: add or adjust `network_policies` entries, binaries, `access`, or `rules`. +5. Edit the YAML: add or adjust `network_policies` entries, binaries, `access`, `rules`, or protocol-specific matchers such as GraphQL operation fields, MCP `method` / `tool` rules, and generic JSON-RPC `method` rules. 6. Push the updated policy when you need a full replacement. Exit codes: 0 = loaded, 1 = validation failed, 124 = timeout. @@ -173,7 +173,7 @@ Use `openshell policy update` when you want to merge network policy changes into - remove one endpoint or one named rule without rewriting the rest of the file. - preview a merged result locally with `--dry-run` before you send it to the gateway. -Use `openshell policy set` instead when you want to replace the full policy, update static sections, or make broader edits that are easier to express in YAML. +Use `openshell policy set` instead when you want to replace the full policy, update static sections, or make broader edits that are easier to express in YAML. Use full YAML for GraphQL, MCP, and JSON-RPC rule shapes. ### Update Commands @@ -210,6 +210,7 @@ This is the practical difference: Current constraints: - `--add-allow` and `--add-deny` work on `protocol: rest` and `protocol: websocket` endpoints. +- GraphQL, MCP, and JSON-RPC fine-grained rules require full policy YAML applied with `openshell policy set`. - `--add-deny` requires the endpoint to already have an allow base, either an `access` preset or explicit allow `rules`. - `protocol: sql` is not a practical incremental workflow today. OpenShell does not do full SQL parsing, and SQL enforcement is not meaningfully supported yet. @@ -228,7 +229,7 @@ Each segment has a fixed meaning: | `host` | Yes | Destination hostname. | | `port` | Yes | Destination port, `1` through `65535`. | | `access` | No | Access preset for L7 endpoints: `read-only`, `read-write`, or `full`. Incremental updates expand presets into protocol-specific method/path rules for REST and WebSocket endpoints. | -| `protocol` | No | L7 inspection mode: `rest`, `websocket`, or `sql`. `sql` is audit-only and not a recommended workflow today. | +| `protocol` | No | L7 inspection mode accepted by `openshell policy update`: `rest`, `websocket`, or `sql`. `sql` is audit-only and not a recommended workflow today. Full policy YAML also supports `graphql`, `mcp`, and `json-rpc`. | | `enforcement` | No | Enforcement mode for inspected traffic: `enforce` or `audit`. | | `options` | No | Comma-separated endpoint options. Use `websocket-credential-rewrite` with `protocol: websocket` or REST compatibility endpoints that perform a WebSocket upgrade. Use `request-body-credential-rewrite` only with `protocol: rest`. | @@ -548,7 +549,7 @@ For an end-to-end walkthrough that combines this policy with a GitHub credential - { path: /usr/bin/gh } ``` -Endpoints with `protocol: rest` enable HTTP request inspection and can opt in to supported text request body credential rewrite. Endpoints with `protocol: websocket` validate WebSocket upgrades and inspect client text messages on the upgraded request path. WebSocket endpoints can also classify GraphQL-over-WebSocket operation messages with the same operation rules used by GraphQL-over-HTTP. Endpoints with `protocol: graphql` parse GraphQL-over-HTTP payloads before evaluating rules. The endpoint-level `path` field lets these protocols share `api.github.com:443` without treating GraphQL payloads as plain REST `POST /graphql` requests. +Endpoints with `protocol: rest` enable HTTP request inspection and can opt in to supported text request body credential rewrite. Endpoints with `protocol: websocket` validate WebSocket upgrades and inspect client text messages on the upgraded request path. WebSocket endpoints can also classify GraphQL-over-WebSocket operation messages with the same operation rules used by GraphQL-over-HTTP. Endpoints with `protocol: graphql` parse GraphQL-over-HTTP payloads before evaluating rules. Endpoints with `protocol: mcp` parse MCP Streamable HTTP request bodies and evaluate `method`, optional `tool`, and supported params rules. Endpoints with `protocol: json-rpc` parse JSON-RPC-over-HTTP request bodies and evaluate `method` rules. The endpoint-level `path` field lets these protocols share `api.github.com:443` without treating GraphQL payloads as plain REST `POST /graphql` requests. @@ -579,6 +580,50 @@ REST rules can also constrain query parameter values: `query` matchers are case-sensitive and run on decoded values. If a request has duplicate keys (for example, `tag=a&tag=b`), every value for that key must match the configured glob(s). +### MCP and JSON-RPC matching + +MCP endpoints use `protocol: mcp`. The proxy parses sandbox-to-server MCP Streamable HTTP request bodies, validates known MCP request and notification params, can evaluate the MCP method against rule `method`, and can match tool calls with the `tool` alias. Unknown extension methods stay addressable as literal method strings. Until OpenShell exposes MCP version profiles, `mcp.allow_all_known_mcp_methods` defaults to `false`, so endpoints require explicit MCP method rules. Set `mcp.allow_all_known_mcp_methods: true` to enable the endpoint method profile; in that mode, rules can omit `method`, and tool selectors are normalized to `tools/call` internally. By default, MCP `tools/call` tool names must match `^[A-Za-z0-9_.-]{1,128}$`; set `mcp.strict_tool_names: false` on that endpoint only when a server intentionally uses names outside the MCP-recommended pattern. Wildcard `tool` matchers require `mcp.strict_tool_names` to remain enabled. Generic JSON-RPC endpoints use `protocol: json-rpc` and evaluate `method`. + +MCP endpoints must declare a concrete destination with `host` and `port` or `ports`. A policy entry that only sets `protocol: mcp` is invalid and is not treated as a wildcard MCP authorization. Use `path: /mcp` when the server's MCP endpoint is path-scoped; omitting `path` matches every HTTP path on that host and port. + +MCP policy enforcement is directional. It applies to HTTP request bodies sent by the sandboxed process to the configured endpoint. JSON-RPC responses and server-to-client MCP messages carried on response bodies or SSE streams are relayed but are not currently parsed for policy enforcement. + +MCP and JSON-RPC endpoint policies currently require full policy YAML applied with `openshell policy set`; the incremental `openshell policy update --add-endpoint` parser does not accept `mcp` or `json-rpc` as protocols. + +```yaml showLineNumbers={false} + mcp_server: + name: mcp_server + endpoints: + - host: mcp.example.com + port: 443 + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + max_body_bytes: 131072 + rules: + - allow: + method: tools/call + tool: read_status + - allow: + method: tools/call + tool: + any: [submit_report, list_reports] + deny_rules: + - method: tools/call + tool: delete_resource + binaries: + - { path: /usr/bin/python3 } +``` + +`mcp.max_body_bytes` controls how many MCP-over-HTTP request body bytes OpenShell buffers for inspection. It defaults to `65536`. `mcp.strict_tool_names` defaults to `true` for each MCP endpoint. `mcp.allow_all_known_mcp_methods` defaults to `false`; when it is unset or `false`, the endpoint must define explicit MCP method rules. If an MCP endpoint sets `mcp.allow_all_known_mcp_methods: true` and omits `rules`, OpenShell allows all MCP-family methods and all tools, then applies any `deny_rules`. A broad allow or deny rule whose method matcher includes `tools/call` cannot be combined with tool-specific allow rules because it would bypass or erase the tool filter; add `tool` or `params.name` to scope `tools/call`, or remove the tool-specific rules. + +Use `protocol: json-rpc` and `method` when you need generic JSON-RPC 2.0 matching for a non-MCP server. Generic JSON-RPC method rules accept exact method names, or `method: "*"` as the all-method sentinel; other wildcard or glob patterns are rejected. `json_rpc.max_body_bytes` controls the generic JSON-RPC inspection buffer. + +Generic JSON-RPC policy `params` matchers are not supported. Generic JSON-RPC policy rules match only the JSON-RPC method. For batch requests, OpenShell evaluates each JSON-RPC call independently and denies the whole batch if any call is denied. + +For MCP, `tool` accepts a string glob or `{ any: [...] }` matcher for `tools/call` `params.name`. Rules that use `tool` or lower-level `params.name` must set `method: tools/call` unless `mcp.allow_all_known_mcp_methods: true` enables the endpoint method profile. MCP method globs are accepted only for the `tools/` method family, such as `tools/*`; omit `method` instead of writing `method: "*"` only when the endpoint method profile should allow all MCP methods. Omit `tool` to allow all tools for a `tools/call` method rule. OpenShell does not support MCP tool argument matching yet; allowed tools accept all argument payloads by default. Other MCP `params` keys are rejected. For batch requests, OpenShell evaluates each JSON-RPC call independently and denies the whole batch if any call is denied. + ### GraphQL matching GraphQL endpoints use `protocol: graphql`. The proxy parses GraphQL-over-HTTP `GET` and `POST` requests, classifies each operation, and evaluates rules against the operation type, optional operation name, and selected root fields. diff --git a/e2e/mcp-conformance.sh b/e2e/mcp-conformance.sh new file mode 100755 index 0000000000..409b5b9f8f --- /dev/null +++ b/e2e/mcp-conformance.sh @@ -0,0 +1,519 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=e2e/support/gateway-common.sh disable=SC1091 +source "${ROOT}/e2e/support/gateway-common.sh" +CONFORMANCE_DIR="${OPENSHELL_MCP_CONFORMANCE_DIR:-${ROOT}/.cache/mcp-conformance}" +# Pinned after v0.1.16 for the upstream tools_call fixture fix. The current +# checkout still needs temporary client-fixture patches for +# modelcontextprotocol/conformance#345; remove patch_conformance_clients when +# OPENSHELL_MCP_CONFORMANCE_REF points at a release containing those fixes. +CONFORMANCE_REF="${OPENSHELL_MCP_CONFORMANCE_REF:-b9041ea41b0188581803459dbae71bc7e02fd995}" +CLIENT_IMAGE="${OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE:-openshell-mcp-conformance-client:local}" +SCENARIOS="${OPENSHELL_MCP_CONFORMANCE_SCENARIOS:-}" +SPEC_VERSION="${OPENSHELL_MCP_CONFORMANCE_SPEC_VERSION:-2025-11-25}" +TIMEOUT_MS="${OPENSHELL_MCP_CONFORMANCE_TIMEOUT_MS:-900000}" +FORCE_REBUILD="${OPENSHELL_MCP_CONFORMANCE_FORCE_REBUILD:-0}" +DOCKER_PULL="${OPENSHELL_MCP_CONFORMANCE_DOCKER_PULL:-0}" +CLIENT_IMAGE_REF_LABEL="org.openshell.mcp-conformance.ref" +CLIENT_IMAGE_DOCKERFILE_LABEL="org.openshell.mcp-conformance.dockerfile" +CLIENT_IMAGE_DOCKERIGNORE_LABEL="org.openshell.mcp-conformance.dockerignore" +CLIENT_IMAGE_FIXTURE_HASH_LABEL="org.openshell.mcp-conformance.fixture-hash" +RUN_SCENARIOS_COMMAND="__openshell_mcp_run_scenarios" +CLIENT_SANDBOX_MANAGED=0 +HOST_BRIDGE_PID="" +HOST_BRIDGE_LOG="" +HOST_BRIDGE_TOKEN="" +RUNNER_CONTAINER_IP="" +RUNNER_CONTAINER="" + +# Static default scenarios for the pinned CONFORMANCE_REF and default +# SPEC_VERSION. To refresh this list after changing either value, list the +# scenarios from the built client image: +# +# docker run --rm openshell-mcp-conformance-client:local \ +# ./node_modules/.bin/tsx src/index.ts list --client --spec-version 2025-11-25 +# +# Then confirm each scenario has a compatible handler in the pinned +# examples/clients/typescript/everything-client.ts. Keep auth/OAuth scenarios +# and the slow sse-retry scenario opt-in unless intentionally broadening the +# default MCP e2e coverage. +DEFAULT_SCENARIOS=( + initialize + tools_call + elicitation-sep1034-client-defaults +) + +require_command() { + local name=$1 + if ! command -v "${name}" >/dev/null 2>&1; then + echo "ERROR: ${name} is required to run MCP conformance e2e tests." >&2 + exit 2 + fi +} + +is_commit_ref() { + [[ "$1" =~ ^[0-9a-fA-F]{40}$ ]] +} + +checkout_conformance() { + mkdir -p "$(dirname "${CONFORMANCE_DIR}")" + + if [ ! -e "${CONFORMANCE_DIR}" ]; then + git init "${CONFORMANCE_DIR}" + git -C "${CONFORMANCE_DIR}" remote add origin \ + https://github.com/modelcontextprotocol/conformance.git + fi + + if [ ! -d "${CONFORMANCE_DIR}/.git" ]; then + echo "ERROR: ${CONFORMANCE_DIR} exists but is not a git checkout." >&2 + echo " Set OPENSHELL_MCP_CONFORMANCE_DIR to another path or remove the directory." >&2 + exit 2 + fi + + if is_commit_ref "${CONFORMANCE_REF}"; then + local current_head="" + current_head="$(git -C "${CONFORMANCE_DIR}" rev-parse HEAD 2>/dev/null || true)" + if [ "${current_head}" = "${CONFORMANCE_REF}" ] \ + && git -C "${CONFORMANCE_DIR}" diff --quiet \ + && git -C "${CONFORMANCE_DIR}" diff --cached --quiet; then + echo "Using cached MCP conformance checkout ${CONFORMANCE_REF}." >&2 + return + fi + fi + + git -C "${CONFORMANCE_DIR}" fetch --depth 1 origin "${CONFORMANCE_REF}" + git -C "${CONFORMANCE_DIR}" checkout --force --detach FETCH_HEAD +} + +docker_image_label() { + local image=$1 + local label=$2 + + docker image inspect \ + --format "{{ index .Config.Labels \"${label}\" }}" \ + "${image}" 2>/dev/null || true +} + +openshell_bin() { + if [ -n "${OPENSHELL_BIN:-}" ]; then + printf '%s\n' "${OPENSHELL_BIN}" + return + fi + + local target_dir + target_dir="$(e2e_cargo_target_dir "${ROOT}")" + printf '%s\n' "${target_dir}/debug/openshell" +} + +patch_conformance_clients() { + node - "${CONFORMANCE_DIR}" <<'NODE' +const fs = require('node:fs'); +const path = require('node:path'); + +const root = process.argv[2]; + +function rewrite(file, rewriter) { + const target = path.join(root, file); + const source = fs.readFileSync(target, 'utf8'); + const next = rewriter(source, file); + + if (next !== source) { + fs.writeFileSync(target, next); + console.error(`Patched upstream MCP conformance fixture: ${file}`); + } +} + +function patchApplyDefaults(source, file) { + if (/elicitation:\s*{\s*form:\s*{\s*applyDefaults:\s*true\s*}\s*}/m.test(source)) { + return source; + } + + const broken = /elicitation:\s*{\s*applyDefaults:\s*true\s*}/m; + if (!broken.test(source)) { + throw new Error(`${file}: could not find the known elicitation defaults fixture`); + } + + return source.replace( + broken, + `elicitation: { + form: { + applyDefaults: true + } + }` + ); +} + +rewrite('examples/clients/typescript/everything-client.ts', (source, file) => { + let next = patchApplyDefaults(source, file); + if (next.includes('elicitation-sep1034-client-defaults')) { + return next; + } + + const oldRegistration = "registerScenario('elicitation-defaults', runElicitationDefaultsClient);"; + const newRegistration = `registerScenarios( + ['elicitation-defaults', 'elicitation-sep1034-client-defaults'], + runElicitationDefaultsClient +);`; + + if (!next.includes(oldRegistration)) { + throw new Error(`${file}: could not find the known elicitation scenario registration`); + } + + return next.replace(oldRegistration, newRegistration); +}); + +rewrite('examples/clients/typescript/elicitation-defaults-test.ts', patchApplyDefaults); +NODE +} + +start_host_bridge() { + local port=$1 + local openshell runner_ip + HOST_BRIDGE_LOG="${ROOT}/.cache/mcp-conformance/host-bridge.log" + mkdir -p "$(dirname "${HOST_BRIDGE_LOG}")" + + if ! openshell="$(openshell_bin)"; then + return 1 + fi + if ! runner_ip="$(runner_container_ip)"; then + return 1 + fi + + RUNNER_CONTAINER_IP="${runner_ip}" + HOST_BRIDGE_TOKEN="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" + OPENSHELL_BIN="${openshell}" \ + OPENSHELL_MCP_CONFORMANCE_RUNNER_IP="${runner_ip}" \ + OPENSHELL_MCP_CONFORMANCE_BRIDGE_TOKEN="${HOST_BRIDGE_TOKEN}" \ + python3 "${ROOT}/e2e/mcp-conformance/host-bridge.py" \ + "${port}" "${ROOT}" "${HOST_BRIDGE_LOG}" & + HOST_BRIDGE_PID=$! + + local deadline + deadline=$((SECONDS + ${OPENSHELL_MCP_CONFORMANCE_HOST_BRIDGE_START_TIMEOUT_SECONDS:-10})) + until python3 - "${port}" <<'PY' +import socket +import sys + +try: + with socket.create_connection(("127.0.0.1", int(sys.argv[1])), timeout=0.2): + pass +except OSError: + raise SystemExit(1) +PY + do + if ! kill -0 "${HOST_BRIDGE_PID}" 2>/dev/null; then + echo "ERROR: MCP conformance host bridge exited before becoming ready (see ${HOST_BRIDGE_LOG})." >&2 + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "ERROR: MCP conformance host bridge did not become ready on 127.0.0.1:${port} (see ${HOST_BRIDGE_LOG})." >&2 + return 1 + fi + sleep 0.1 + done +} + +# Resolve the hostname the runner container uses to reach the host bridge. +# In CI, e2e/with-docker-gateway.sh connects the job container (which hosts the +# bridge) to the e2e Docker network with the host.openshell.internal alias. On +# local Docker Desktop, host.docker.internal reaches the host. On local Linux, +# the runner container is started with --add-host ...:host-gateway. +host_bridge_hostname() { + if [ -n "${OPENSHELL_MCP_CONFORMANCE_HOST_BRIDGE_HOSTNAME:-}" ]; then + printf '%s\n' "${OPENSHELL_MCP_CONFORMANCE_HOST_BRIDGE_HOSTNAME}" + return + fi + + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '%s\n' "host.openshell.internal" + elif [ "$(uname -s)" = "Darwin" ]; then + printf '%s\n' "host.docker.internal" + else + printf '%s\n' "host.openshell.internal" + fi +} + +runner_container_ip() { + local network ip + + network="${OPENSHELL_E2E_DOCKER_NETWORK_NAME:-${OPENSHELL_E2E_NETWORK_NAME:-}}" + if [ -z "${network}" ]; then + echo "ERROR: no e2e Docker network resolved for the MCP conformance runner container." >&2 + return 1 + fi + if [ -z "${RUNNER_CONTAINER}" ]; then + echo "ERROR: MCP conformance runner container has not been started." >&2 + return 1 + fi + + ip="$(docker inspect \ + --format "{{with index .NetworkSettings.Networks \"${network}\"}}{{.IPAddress}}{{end}}" \ + "${RUNNER_CONTAINER}")" + if [ -z "${ip}" ]; then + echo "ERROR: failed to resolve MCP conformance runner IP on Docker network ${network}." >&2 + return 1 + fi + printf '%s\n' "${ip}" +} + +stop_host_bridge() { + if [ -n "${HOST_BRIDGE_PID}" ] && kill -0 "${HOST_BRIDGE_PID}" 2>/dev/null; then + kill "${HOST_BRIDGE_PID}" 2>/dev/null || true + wait "${HOST_BRIDGE_PID}" 2>/dev/null || true + fi + HOST_BRIDGE_PID="" + HOST_BRIDGE_TOKEN="" + RUNNER_CONTAINER_IP="" +} + +build_client_image() { + local conformance_head dockerfile_hash dockerignore_hash fixture_hash + local image_ref image_dockerfile image_dockerignore image_fixture_hash + local -a pull_args=() + + conformance_head="$(git -C "${CONFORMANCE_DIR}" rev-parse HEAD)" + dockerfile_hash="$(git -C "${ROOT}" hash-object "${ROOT}/e2e/mcp-conformance/Dockerfile.client")" + dockerignore_hash="$(git -C "${ROOT}" hash-object "${ROOT}/e2e/mcp-conformance/Dockerfile.client.dockerignore")" + fixture_hash="$( + git -C "${CONFORMANCE_DIR}" diff -- \ + examples/clients/typescript/everything-client.ts \ + examples/clients/typescript/elicitation-defaults-test.ts | + git hash-object --stdin + )" + + image_ref="$(docker_image_label "${CLIENT_IMAGE}" "${CLIENT_IMAGE_REF_LABEL}")" + image_dockerfile="$(docker_image_label "${CLIENT_IMAGE}" "${CLIENT_IMAGE_DOCKERFILE_LABEL}")" + image_dockerignore="$(docker_image_label "${CLIENT_IMAGE}" "${CLIENT_IMAGE_DOCKERIGNORE_LABEL}")" + image_fixture_hash="$(docker_image_label "${CLIENT_IMAGE}" "${CLIENT_IMAGE_FIXTURE_HASH_LABEL}")" + if [ "${FORCE_REBUILD}" != "1" ] \ + && [ "${image_ref}" = "${conformance_head}" ] \ + && [ "${image_dockerfile}" = "${dockerfile_hash}" ] \ + && [ "${image_dockerignore}" = "${dockerignore_hash}" ] \ + && [ "${image_fixture_hash}" = "${fixture_hash}" ]; then + echo "Using cached MCP conformance client image ${CLIENT_IMAGE} (${conformance_head})." >&2 + return + fi + + if [ "${DOCKER_PULL}" = "1" ]; then + pull_args=(--pull) + fi + + docker build "${pull_args[@]}" \ + --label "${CLIENT_IMAGE_REF_LABEL}=${conformance_head}" \ + --label "${CLIENT_IMAGE_DOCKERFILE_LABEL}=${dockerfile_hash}" \ + --label "${CLIENT_IMAGE_DOCKERIGNORE_LABEL}=${dockerignore_hash}" \ + --label "${CLIENT_IMAGE_FIXTURE_HASH_LABEL}=${fixture_hash}" \ + -f "${ROOT}/e2e/mcp-conformance/Dockerfile.client" \ + -t "${CLIENT_IMAGE}" \ + "${CONFORMANCE_DIR}" +} + +create_client_sandbox() { + if [ -n "${OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX:-}" ]; then + echo "Using existing MCP conformance client sandbox ${OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX}." >&2 + return + fi + + local sandbox_name policy_file openshell + sandbox_name="openshell-mcp-client-$$" + policy_file="$(mktemp "${TMPDIR:-/tmp}/openshell-mcp-conformance-base-policy.XXXXXX.yaml")" + openshell="$(openshell_bin)" + + # The upstream runner binds its per-scenario test server with listen(0), and + # the port can be outside the OS ephemeral range. Create the reusable sandbox + # with a harmless placeholder; the client wrapper installs the exact policy + # for each scenario URL before executing the TypeScript client. + python3 "${ROOT}/e2e/mcp-conformance/render-policy.py" \ + "http://192.0.2.1:1/" "${policy_file}" \ + "${ROOT}/e2e/mcp-conformance/policy-template.yaml" >/dev/null + + echo "Creating MCP conformance client sandbox ${sandbox_name}..." >&2 + if ! "${openshell}" sandbox create \ + --name "${sandbox_name}" \ + --from "${CLIENT_IMAGE}" \ + --policy "${policy_file}" \ + --no-tty \ + -- true; then + rm -f "${policy_file}" + return 1 + fi + rm -f "${policy_file}" + + export OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX="${sandbox_name}" + export OPENSHELL_MCP_CONFORMANCE_POLICY_WAIT="${OPENSHELL_MCP_CONFORMANCE_POLICY_WAIT:-1}" + CLIENT_SANDBOX_MANAGED=1 +} + +cleanup_client_sandbox() { + if [ "${CLIENT_SANDBOX_MANAGED}" != "1" ]; then + return + fi + + local openshell + openshell="$(openshell_bin)" + echo "Deleting MCP conformance client sandbox ${OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX}..." >&2 + "${openshell}" sandbox delete "${OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX}" >/dev/null 2>&1 || true +} + +# Start the upstream conformance runner in a plain Docker container on the e2e +# network. The runner runs node (and the bundled MCP test server) off the host +# for isolation, but unlike an OpenShell sandbox it has an ordinary, +# externally-routable network address: its listen(0) test server is reachable +# from the client sandbox, and it can call the host bridge back directly. +create_runner_container() { + local network + local -a add_host_args=() + + network="${OPENSHELL_E2E_DOCKER_NETWORK_NAME:-${OPENSHELL_E2E_NETWORK_NAME:-}}" + if [ -z "${network}" ]; then + echo "ERROR: no e2e Docker network resolved for the MCP conformance runner container." >&2 + return 1 + fi + + RUNNER_CONTAINER="openshell-mcp-runner-$$" + + # On local Linux the host bridge runs on the host, so map the bridge hostnames + # to the Docker host gateway. CI (job-container network alias) and Docker + # Desktop resolve these names without an explicit mapping. + if [ "${GITHUB_ACTIONS:-}" != "true" ] && [ "$(uname -s)" != "Darwin" ]; then + add_host_args=(--add-host "host.openshell.internal:host-gateway" --add-host "host.docker.internal:host-gateway") + fi + + echo "Starting MCP conformance runner container ${RUNNER_CONTAINER} on Docker network ${network}..." >&2 + if ! docker run -d --rm \ + --name "${RUNNER_CONTAINER}" \ + --network "${network}" \ + "${add_host_args[@]}" \ + "${CLIENT_IMAGE}" \ + sleep infinity >/dev/null; then + RUNNER_CONTAINER="" + return 1 + fi + + if ! docker cp "${ROOT}/e2e/mcp-conformance/runner-shim.mjs" "${RUNNER_CONTAINER}:/tmp/openshell-mcp-runner-shim.mjs" \ + || ! docker cp "${ROOT}/e2e/mcp-conformance/expected-failures.yml" "${RUNNER_CONTAINER}:/tmp/expected-failures.yml"; then + return 1 + fi +} + +cleanup_runner_container() { + if [ -n "${RUNNER_CONTAINER}" ]; then + docker rm -f "${RUNNER_CONTAINER}" >/dev/null 2>&1 || true + fi + RUNNER_CONTAINER="" +} + +scenario_list_for_args() { + local scenario_list + local -a scenario_args=("$@") + + if [ "${#scenario_args[@]}" -gt 0 ]; then + scenario_list="${scenario_args[*]}" + elif [ -n "${SCENARIOS}" ]; then + scenario_list="${SCENARIOS}" + else + scenario_list="${DEFAULT_SCENARIOS[*]}" + fi + + printf '%s\n' "${scenario_list}" +} + +run_scenarios_in_runner_container() { + local bridge_host bridge_port scenario scenario_list + local -a passed=() + local -a failed=() + + scenario_list="$(scenario_list_for_args "$@")" + if [ -z "${scenario_list}" ]; then + echo "ERROR: no MCP conformance scenarios resolved." >&2 + return 2 + fi + + bridge_port="$(e2e_pick_port)" + create_client_sandbox || return 1 + create_runner_container || return 1 + start_host_bridge "${bridge_port}" || return 1 + + bridge_host="$(host_bridge_hostname)" + echo "MCP conformance host bridge callback: http://${bridge_host}:${bridge_port}/run" >&2 + + for scenario in ${scenario_list}; do + echo "=== MCP conformance: ${scenario} ===" + # shellcheck disable=SC2016 + if docker exec \ + --env "MCP_CONFORMANCE_HOST_BRIDGE_URL=http://${bridge_host}:${bridge_port}/run" \ + --env "MCP_CONFORMANCE_HOST_BRIDGE_TOKEN=${HOST_BRIDGE_TOKEN}" \ + --env "MCP_CONFORMANCE_RUNNER_IP=${RUNNER_CONTAINER_IP}" \ + "${RUNNER_CONTAINER}" \ + sh -c 'cd /opt/mcp-conformance && exec node dist/index.js client --command "node /tmp/openshell-mcp-runner-shim.mjs" --scenario "$1" --spec-version "$2" --expected-failures "$3" --timeout "$4"' \ + sh "${scenario}" "${SPEC_VERSION}" "/tmp/expected-failures.yml" "${TIMEOUT_MS}" \ + }" + echo "Failed (${#failed[@]}): ${failed[*]:-}" + + if [ "${#failed[@]}" -ne 0 ]; then + return 1 + fi +} + +cleanup_scenario_resources() { + cleanup_runner_container + stop_host_bridge + cleanup_client_sandbox +} + +run_scenarios_with_client_sandbox() { + # Tear down the runner container, host bridge, and client sandbox on any exit, + # including Ctrl-C / SIGTERM. The cleanups are no-ops if their resource was + # never created, so an early failure is safe too. + trap cleanup_scenario_resources EXIT + trap 'exit 130' INT TERM + + run_scenarios_in_runner_container "$@" +} + +run_scenarios_under_gateway() { + export OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE="${CLIENT_IMAGE}" + export OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE="${OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE:-${CLIENT_IMAGE}}" + + "${ROOT}/e2e/with-docker-gateway.sh" bash "${BASH_SOURCE[0]}" "${RUN_SCENARIOS_COMMAND}" "$@" +} + +main() { + cd "${ROOT}" + + if [ "${1:-}" = "${RUN_SCENARIOS_COMMAND}" ]; then + shift + run_scenarios_with_client_sandbox "$@" + return + fi + + # git fetches and pins the upstream conformance repo; node patches temporary + # fixture drift before the image build; docker builds and runs the + # runner/client containers; python3 runs the host bridge and renders policies. + # npm and tsx run inside the container image, not on the host. + require_command git + require_command node + require_command docker + require_command python3 + + echo "MCP conformance spec version: ${SPEC_VERSION}" >&2 + checkout_conformance + patch_conformance_clients + build_client_image + run_scenarios_under_gateway "$@" +} + +main "$@" diff --git a/e2e/mcp-conformance/Dockerfile.client b/e2e/mcp-conformance/Dockerfile.client new file mode 100644 index 0000000000..e40e23df52 --- /dev/null +++ b/e2e/mcp-conformance/Dockerfile.client @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +FROM public.ecr.aws/docker/library/node:22-bookworm-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates iproute2 \ + && rm -rf /var/lib/apt/lists/* + +ARG SANDBOX_UID=1000660000 +ARG SANDBOX_GID=1000660000 + +# Match the sandbox user expected by OpenShell policies and supervisor setup. +# The UID/GID are intentionally outside Debian's default login.defs range. +RUN groupadd -K "GID_MAX=${SANDBOX_GID}" -g "${SANDBOX_GID}" sandbox \ + && useradd -K "UID_MAX=${SANDBOX_UID}" --no-log-init -m -u "${SANDBOX_UID}" -g sandbox sandbox + +WORKDIR /opt/mcp-conformance + +COPY . . +RUN if [ -f package-lock.json ]; then npm ci; else npm install; fi +RUN npm run build +RUN chown -R sandbox:sandbox /opt/mcp-conformance /home/sandbox + +USER sandbox +CMD ["sleep", "infinity"] diff --git a/e2e/mcp-conformance/Dockerfile.client.dockerignore b/e2e/mcp-conformance/Dockerfile.client.dockerignore new file mode 100644 index 0000000000..d0ef9a556d --- /dev/null +++ b/e2e/mcp-conformance/Dockerfile.client.dockerignore @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +.git +dist +node_modules +.openshell-mcp-conformance-*.stamp diff --git a/e2e/mcp-conformance/README.md b/e2e/mcp-conformance/README.md new file mode 100644 index 0000000000..398decaa87 --- /dev/null +++ b/e2e/mcp-conformance/README.md @@ -0,0 +1,82 @@ +# MCP Conformance E2E + +This directory contains the OpenShell wrapper for the upstream +`modelcontextprotocol/conformance` runner. + +The workflow checks out and builds the upstream conformance repository, then +runs its CLI in client mode. To keep the untrusted upstream node runner off the +host, the wrapper runs it inside a plain Docker container on the e2e Docker +network (not an OpenShell sandbox, which is egress-only and could not accept the +client's inbound connection). The upstream runner starts a real MCP test server +and invokes its client command — `runner-shim.mjs` — with that server URL. + +`runner-shim.mjs` stands in for the MCP client: instead of speaking MCP itself, +it posts the server URL back to the host bridge (`host-bridge.py`) over HTTP. The +host bridge runs `client-through-openshell.sh`, which runs the upstream +TypeScript `everything-client` inside an OpenShell client sandbox for each +scenario, so the MCP traffic crosses the sandbox proxy. A single Docker-backed +OpenShell e2e gateway and one reusable client sandbox serve the whole scenario +list. The runner deliberately has no gateway credentials; keeping the privileged +client launch on `host-bridge.py` is the trust boundary. The harness gives the +runner a per-run bridge capability and gives the bridge the runner container IP. +The bridge only accepts requests with that capability, only renders server URLs +whose host is the runner container IP, only forwards the MCP conformance +scenario environment allowlist, and starts the client wrapper with a small host +environment allowlist instead of inheriting token-bearing host environment +variables. It does not use the HTTP peer source address as the runner identity, +because Docker NAT can make legitimate callbacks appear to come from a gateway +address. + +The upstream runner reports its test server URL as `localhost`. The runner +container has an ordinary, externally-routable address on the e2e network, so +`runner-shim.mjs` rewrites `localhost` to that container's IP — which the client +sandbox can reach through its egress proxy. The runner container reaches the host +bridge at `host.openshell.internal` (the alias `e2e/with-docker-gateway.sh` +attaches to the CI job container on the e2e network), at `host.docker.internal` +on local Docker Desktop, or via `--add-host ...:host-gateway` on local Linux. + +The generated policy uses `protocol: mcp` and sets +`mcp.allow_all_known_mcp_methods: true` so omitted rule methods use the endpoint +MCP method profile. That keeps OpenShell deny-by-default at the network boundary +while allowing the upstream scenarios to exercise MCP behavior. The policy body +lives in `policy-template.yaml`; the wrapper renders its host, port, and path +placeholders from the upstream server URL. + +For local runs, the wrapper builds `openshell/supervisor:dev` automatically +when no supervisor image override is set. Set `OPENSHELL_DOCKER_SUPERVISOR_IMAGE` +or `OPENSHELL_SUPERVISOR_IMAGE` to use a prebuilt pullable image instead. + +The pinned upstream checkout includes reference-client fixture drift that is +tracked in `modelcontextprotocol/conformance#345`. The wrapper patches the +checkout before building the client image so the bundled TypeScript client +advertises `elicitation.form.applyDefaults` and accepts the canonical +`elicitation-sep1034-client-defaults` scenario. It also routes `sse-retry` to +the upstream standalone `sse-retry-test.ts` client so the reconnect timing path +is exercised instead of aliasing it to another scenario. + +Remove those local workarounds when `OPENSHELL_MCP_CONFORMANCE_REF` points at +an upstream release that includes the `#345` fixes. + +When enabling broader upstream suites, add scenarios that OpenShell does not yet +support through the MCP proxy to `expected-failures.yml`. The upstream +runner treats listed failures as allowed and treats stale entries as failures. +The default run uses a static scenario list in `e2e/mcp-conformance.sh`. To +refresh it after changing the pinned upstream ref or default spec, list the +scenarios from the built client image: + +```shell +docker run --rm openshell-mcp-conformance-client:local \ + ./node_modules/.bin/tsx src/index.ts list --client --spec-version 2025-11-25 +``` + +Then confirm each scenario has a compatible handler in the pinned +`examples/clients/typescript/everything-client.ts`. The default list skips +opt-in scenarios, including auth/OAuth flows and the slow `sse-retry` scenario. +Set `OPENSHELL_MCP_CONFORMANCE_SCENARIOS=sse-retry` or pass `sse-retry` as an +argument to run it explicitly. + +The wrapper caches the pinned upstream checkout, the local conformance runner +build, and the Docker client image. Set +`OPENSHELL_MCP_CONFORMANCE_FORCE_REBUILD=1` to refresh those build artifacts, or +`OPENSHELL_MCP_CONFORMANCE_DOCKER_PULL=1` to pull the client image base during a +rebuild. diff --git a/e2e/mcp-conformance/client-through-openshell.sh b/e2e/mcp-conformance/client-through-openshell.sh new file mode 100755 index 0000000000..f0f9d1d4e2 --- /dev/null +++ b/e2e/mcp-conformance/client-through-openshell.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Runs the upstream MCP conformance client in an OpenShell-managed sandbox. +# +# The modelcontextprotocol/conformance runner runs in a separate container and +# posts the URL of its MCP test server to a host bridge, which invokes this +# script with that URL. The parent harness creates one reusable conformance +# client sandbox for the whole scenario list before this script is invoked. This +# wrapper verifies the active gateway is reachable, applies the per-scenario +# server policy to that sandbox, and runs the upstream TypeScript +# everything-client inside it so its MCP traffic crosses the sandbox proxy. + +set -euo pipefail + +usage() { + echo "usage: $0 " >&2 +} + +if [ "$#" -ne 1 ]; then + usage + exit 2 +fi + +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" + +# shellcheck source=e2e/support/gateway-common.sh disable=SC1091 +source "${ROOT}/e2e/support/gateway-common.sh" +if [ -z "${OPENSHELL_BIN:-}" ]; then + TARGET_DIR="$(e2e_cargo_target_dir "${ROOT}")" + OPENSHELL_BIN="${TARGET_DIR}/debug/openshell" +fi + +require_active_gateway() { + local status_output + + if ! status_output="$("${OPENSHELL_BIN}" status 2>&1)"; then + echo "ERROR: no reachable active OpenShell gateway for MCP conformance." >&2 + echo " Run e2e/mcp-conformance.sh so it starts one shared Docker-backed gateway." >&2 + echo "=== openshell status output ===" >&2 + printf '%s\n' "${status_output}" >&2 + echo "=== end openshell status output ===" >&2 + exit 2 + fi +} + +SERVER_URL="$1" +CLIENT_SANDBOX="${OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX:?set OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX to the reusable conformance client sandbox name}" +POLICY_TEMPLATE="${ROOT}/e2e/mcp-conformance/policy-template.yaml" +POLICY_WAIT="${OPENSHELL_MCP_CONFORMANCE_POLICY_WAIT:-0}" +POLICY_WAIT_TIMEOUT="${OPENSHELL_MCP_CONFORMANCE_POLICY_WAIT_TIMEOUT:-60}" +CLIENT_TIMEOUT_SECONDS="${OPENSHELL_MCP_CONFORMANCE_CLIENT_TIMEOUT_SECONDS:-120}" + +require_active_gateway + +POLICY_FILE="$(mktemp "${TMPDIR:-/tmp}/openshell-mcp-conformance-policy.XXXXXX.yaml")" +trap 'rm -f "${POLICY_FILE}"' EXIT + +CLIENT_SERVER_URL="$(python3 "${ROOT}/e2e/mcp-conformance/render-policy.py" "${SERVER_URL}" "${POLICY_FILE}" "${POLICY_TEMPLATE}")" + +ENV_ARGS=() + +# These environment variables are set by the upstream conformance test runner +# before it invokes the configured client command. Forward them into the +# sandbox because the sandboxed TypeScript client depends on them to select the +# scenario and read scenario-specific context. +for NAME in MCP_CONFORMANCE_SCENARIO MCP_CONFORMANCE_CONTEXT MCP_CONFORMANCE_PROTOCOL_VERSION; do + if [ -n "${!NAME+x}" ]; then + ENV_ARGS+=(--env "${NAME}=${!NAME}") + fi +done + +POLICY_SET_COMMAND=( + "${OPENSHELL_BIN}" policy set "${CLIENT_SANDBOX}" + --policy "${POLICY_FILE}" +) +if [ "${POLICY_WAIT}" = "1" ]; then + POLICY_SET_COMMAND+=(--wait --timeout "${POLICY_WAIT_TIMEOUT}") +fi +"${POLICY_SET_COMMAND[@]}" + +# Exec request validation rejects newline/control characters in command +# arguments, so keep the sandbox-side script as a single argument without +# embedded newlines. +# shellcheck disable=SC2016 +SANDBOX_CLIENT_SCRIPT='cd /opt/mcp-conformance; case "${MCP_CONFORMANCE_SCENARIO:-}" in tools_call|tools-call) client=examples/clients/typescript/test2.ts ;; sse-retry) client=examples/clients/typescript/sse-retry-test.ts ;; *) client=examples/clients/typescript/everything-client.ts ;; esac; exec ./node_modules/.bin/tsx "$client" "$1"' + +SANDBOX_COMMAND=( + "${OPENSHELL_BIN}" sandbox exec + --name "${CLIENT_SANDBOX}" + --no-tty + --timeout "${CLIENT_TIMEOUT_SECONDS}" + "${ENV_ARGS[@]}" + -- + sh -c "${SANDBOX_CLIENT_SCRIPT}" \ + sh "${CLIENT_SERVER_URL}" +) + +"${SANDBOX_COMMAND[@]}" +""" + +import hmac +import json +import os +import subprocess +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from ipaddress import ip_address +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +PORT = int(sys.argv[1]) +ROOT = Path(sys.argv[2]) +LOG_PATH = Path(sys.argv[3]) +TIMEOUT = ( + int(os.environ.get("OPENSHELL_MCP_CONFORMANCE_CLIENT_TIMEOUT_SECONDS", "120")) + 30 +) +REQUEST_BODY_TIMEOUT_SECONDS = 10 +MAX_REQUEST_BODY_BYTES = 256 * 1024 +TOKEN_HEADER = "x-openshell-mcp-conformance-token" +BRIDGE_TOKEN = os.environ["OPENSHELL_MCP_CONFORMANCE_BRIDGE_TOKEN"] +RUNNER_IP = os.environ["OPENSHELL_MCP_CONFORMANCE_RUNNER_IP"] +ALLOWED_CONFORMANCE_ENV = frozenset( + { + "MCP_CONFORMANCE_SCENARIO", + "MCP_CONFORMANCE_CONTEXT", + "MCP_CONFORMANCE_PROTOCOL_VERSION", + } +) +HOST_ENV_ALLOWLIST = frozenset( + { + "CARGO_HOME", + "CARGO_TARGET_DIR", + "HOME", + "LANG", + "LC_ALL", + "MISE_CACHE_DIR", + "MISE_CONFIG_DIR", + "MISE_DATA_DIR", + "MISE_STATE_DIR", + "OPENSHELL_BIN", + "OPENSHELL_GATEWAY", + "OPENSHELL_MCP_CONFORMANCE_CLIENT_SANDBOX", + "OPENSHELL_MCP_CONFORMANCE_CLIENT_TIMEOUT_SECONDS", + "OPENSHELL_MCP_CONFORMANCE_POLICY_WAIT", + "OPENSHELL_MCP_CONFORMANCE_POLICY_WAIT_TIMEOUT", + "OPENSHELL_PROVISION_TIMEOUT", + "PATH", + "RUSTUP_HOME", + "TMP", + "TEMP", + "TMPDIR", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + "XDG_STATE_HOME", + } +) + + +def log(message: str) -> None: + with LOG_PATH.open("a", encoding="utf-8") as fh: + fh.write(message + "\n") + + +def canonical_ip(value: str): + parsed = ip_address(value) + return getattr(parsed, "ipv4_mapped", None) or parsed + + +def captured_text(value: str | bytes | None) -> str: + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + return value + + +def subprocess_env( + payload_env: dict[str, str], expected_server_host: str +) -> dict[str, str]: + env = {name: os.environ[name] for name in HOST_ENV_ALLOWLIST if name in os.environ} + env.update(payload_env) + env["OPENSHELL_MCP_CONFORMANCE_EXPECTED_SERVER_HOST"] = expected_server_host + return env + + +class Handler(BaseHTTPRequestHandler): + def setup(self) -> None: + super().setup() + self.connection.settimeout(REQUEST_BODY_TIMEOUT_SECONDS) + + def send_json(self, status: int, body: dict[str, object]) -> None: + encoded = json.dumps(body).encode("utf-8") + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(encoded))) + self.send_header("connection", "close") + self.end_headers() + self.wfile.write(encoded) + + def reject(self, status: int, detail: str) -> None: + self.close_connection = True + log(f"rejecting bridge request from {self.client_address[0]}: {detail}") + self.send_json(status, {"error": "invalid_bridge_request", "detail": detail}) + + def bridge_token_valid(self) -> bool: + supplied = self.headers.get(TOKEN_HEADER, "") + return hmac.compare_digest(supplied, BRIDGE_TOKEN) + + def request_body_length(self) -> int | None: + raw_length = self.headers.get("content-length") + if raw_length is None: + self.reject(411, "content-length is required") + return None + try: + length = int(raw_length) + except ValueError: + self.reject(400, "content-length must be an integer") + return None + if length < 0: + self.reject(400, "content-length must not be negative") + return None + if length > MAX_REQUEST_BODY_BYTES: + self.reject(413, "request body is too large") + return None + return length + + def read_request_payload(self, length: int) -> dict[str, Any] | None: + try: + raw_body = self.rfile.read(length) + except TimeoutError: + self.reject(408, "timed out reading request body") + return None + if len(raw_body) != length: + self.reject(400, "request body ended before content-length") + return None + try: + payload = json.loads(raw_body) + except json.JSONDecodeError as err: + self.reject(400, str(err)) + return None + if not isinstance(payload, dict): + self.reject(400, "request body must be a JSON object") + return None + return payload + + def do_POST(self) -> None: + if self.path != "/run": + self.send_response(404) + self.end_headers() + return + + if not self.bridge_token_valid(): + self.reject(403, "invalid bridge capability") + return + + length = self.request_body_length() + if length is None: + return + + payload = self.read_request_payload(length) + if payload is None: + return + + server_url = payload.get("server_url") + if not isinstance(server_url, str): + self.reject(400, "server_url must be a string") + return + + parsed = urlparse(server_url) + if parsed.scheme not in {"http", "https"}: + self.reject(403, "server_url scheme must be http or https") + return + + try: + target_ip = canonical_ip(parsed.hostname or "") + expected_ip = canonical_ip(RUNNER_IP) + except ValueError: + self.reject(403, "server_url host must match the runner container IP") + return + + if target_ip != expected_ip: + self.reject(403, "server_url host must match the runner container IP") + return + + payload_env = payload.get("env", {}) + if not isinstance(payload_env, dict): + self.reject(400, "env must be an object") + return + if any(name not in ALLOWED_CONFORMANCE_ENV for name in payload_env): + self.reject(403, "env contains unsupported keys") + return + if any(not isinstance(value, str) for value in payload_env.values()): + self.reject(400, "env values must be strings") + return + + env = subprocess_env(payload_env, str(expected_ip)) + log(f"running client for {server_url}") + try: + result = subprocess.run( + ["bash", "e2e/mcp-conformance/client-through-openshell.sh", server_url], + cwd=ROOT, + env=env, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=TIMEOUT, + ) + log(f"client exited {result.returncode} for {server_url}") + body = { + "exit_code": result.returncode, + "stdout": result.stdout, + "stderr": result.stderr, + } + except subprocess.TimeoutExpired as err: + body = { + "exit_code": 124, + "stdout": captured_text(err.stdout), + "stderr": captured_text(err.stderr) + + f"\nhost bridge timed out after {TIMEOUT}s\n", + } + self.send_json(200, body) + + def log_message(self, format: str, *args: Any) -> None: + log(format % args) + + +def main() -> None: + server = ThreadingHTTPServer(("0.0.0.0", PORT), Handler) + log(f"host bridge listening on {PORT}") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/e2e/mcp-conformance/policy-template.yaml b/e2e/mcp-conformance/policy-template.yaml new file mode 100644 index 0000000000..2344bff0c5 --- /dev/null +++ b/e2e/mcp-conformance/policy-template.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /bin + - /usr + - /lib + - /lib64 + - /proc + - /sys + - /dev/urandom + - /etc + - /opt + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + - /home/sandbox + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + mcp_conformance: + name: mcp_conformance + endpoints: + - ${host_spec} +${port_spec} + path: ${path} + protocol: mcp + enforcement: enforce + allowed_ips: + - "10.0.0.0/8" + - "172.16.0.0/12" + - "192.168.0.0/16" + - "fc00::/7" + mcp: + max_body_bytes: 131072 + allow_all_known_mcp_methods: true + rules: + - allow: {} + binaries: + - path: /bin/sh + - path: /usr/bin/env + - path: /usr/local/bin/node + - path: /usr/bin/node + - path: /opt/mcp-conformance/node_modules/.bin/* diff --git a/e2e/mcp-conformance/render-policy.py b/e2e/mcp-conformance/render-policy.py new file mode 100644 index 0000000000..6f7a0db923 --- /dev/null +++ b/e2e/mcp-conformance/render-policy.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Render the OpenShell client policy for a conformance server URL. + +Parses the server URL into host/port/path, substitutes them into +policy-template.yaml, writes the rendered policy, and prints the (possibly +rewritten) URL the client should connect to. Used both to seed the reusable +client sandbox with a placeholder policy and to install the per-scenario policy +in client-through-openshell.sh. + +Usage: render-policy.py +""" + +import json +import os +import string +import sys +from ipaddress import ip_address +from pathlib import Path +from urllib.parse import urlparse, urlunparse + +raw_url, policy_file, policy_template = sys.argv[1:4] +parsed = urlparse(raw_url) + +if parsed.scheme not in ("http", "https"): + raise SystemExit(f"unsupported conformance server URL scheme: {parsed.scheme!r}") + +host = parsed.hostname +if not host: + raise SystemExit(f"conformance server URL is missing a host: {raw_url}") + +expected_host = os.environ.get("OPENSHELL_MCP_CONFORMANCE_EXPECTED_SERVER_HOST") +if expected_host: + try: + actual_ip = ip_address(host) + actual_ip = getattr(actual_ip, "ipv4_mapped", None) or actual_ip + expected_ip = ip_address(expected_host) + expected_ip = getattr(expected_ip, "ipv4_mapped", None) or expected_ip + except ValueError as err: + raise SystemExit( + "conformance server URL host must be an IP address when " + "OPENSHELL_MCP_CONFORMANCE_EXPECTED_SERVER_HOST is set" + ) from err + if actual_ip != expected_ip: + raise SystemExit( + f"conformance server URL host {actual_ip} does not match expected " + f"runner host {expected_ip}" + ) + +target_host = ( + "host.openshell.internal" if host in {"localhost", "127.0.0.1", "::1"} else host +) +port = parsed.port or (443 if parsed.scheme == "https" else 80) +path = parsed.path or "/" +netloc_host = ( + f"[{target_host}]" + if ":" in target_host and not target_host.startswith("[") + else target_host +) +netloc = f"{netloc_host}:{port}" +rewritten = urlunparse( + (parsed.scheme, netloc, path, parsed.params, parsed.query, parsed.fragment) +) + +template = string.Template(Path(policy_template).read_text(encoding="utf-8")) +policy = template.substitute( + host_spec=f"host: {json.dumps(target_host)}", + port_spec=f" port: {port}", + path=json.dumps(path), +) +Path(policy_file).write_text(policy, encoding="utf-8") + +print(rewritten) diff --git a/e2e/mcp-conformance/runner-shim.mjs b/e2e/mcp-conformance/runner-shim.mjs new file mode 100644 index 0000000000..41a4423f89 --- /dev/null +++ b/e2e/mcp-conformance/runner-shim.mjs @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Client command for the upstream MCP conformance runner, executed inside the +// runner container. +// +// Why this indirection exists: the conformance runner runs untrusted upstream +// node off the host, so it deliberately has no openshell binary and no gateway +// credentials. But the MCP client under test must run inside an OpenShell +// sandbox so its traffic crosses the policy-enforced proxy (the whole point of +// the e2e). This script bridges that gap without widening the runner's +// privileges: instead of running the MCP client itself, it posts the test +// server URL to the host bridge, which holds the gateway credentials and runs +// the real client in a sandbox, then returns its result. The runner can only +// ask "run a client against this URL" — it cannot touch the gateway control +// plane. + +import http from 'node:http'; +import os from 'node:os'; +import process from 'node:process'; + +// Resolve the routable IPv4 address of the runner container on the e2e Docker +// network. The conformance runner reports its test server URL as localhost; the +// client sandbox connects from a different container, so localhost must be +// rewritten to this container's network address. +function runnerAddress() { + if (process.env.MCP_CONFORMANCE_RUNNER_IP) { + return process.env.MCP_CONFORMANCE_RUNNER_IP; + } + for (const addrs of Object.values(os.networkInterfaces())) { + for (const addr of addrs ?? []) { + if (addr.family === 'IPv4' && !addr.internal) { + return addr.address; + } + } + } + throw new Error('failed to resolve runner container IPv4 address'); +} + +function rewriteServerUrl(rawUrl) { + const url = new URL(rawUrl); + if (['localhost', '127.0.0.1', '[::1]', '::1'].includes(url.hostname)) { + url.hostname = runnerAddress(); + } + return url.toString(); +} + +const bridgeUrl = process.env.MCP_CONFORMANCE_HOST_BRIDGE_URL; +if (!bridgeUrl) { + throw new Error('MCP_CONFORMANCE_HOST_BRIDGE_URL is required'); +} +const bridgeToken = process.env.MCP_CONFORMANCE_HOST_BRIDGE_TOKEN; +if (!bridgeToken) { + throw new Error('MCP_CONFORMANCE_HOST_BRIDGE_TOKEN is required'); +} +const parsedBridgeUrl = new URL(bridgeUrl); +if (parsedBridgeUrl.protocol !== 'http:') { + throw new Error(`bridge URL must use http:, got ${parsedBridgeUrl.protocol}`); +} +const serverUrl = process.argv[2]; +if (!serverUrl) { + throw new Error('usage: node runner-shim.mjs '); +} + +// POST the rewritten server URL to the host bridge, which runs the real MCP +// client inside an OpenShell sandbox and returns its result. The runner is a +// plain container with ordinary egress, so this is a direct HTTP call. +function postJson(url, payload) { + const body = Buffer.from(JSON.stringify(payload), 'utf8'); + const timeoutMs = Number.parseInt(process.env.MCP_CONFORMANCE_HOST_BRIDGE_TIMEOUT_MS ?? '600000', 10); + + function snippet(raw) { + return raw.length > 4096 ? `${raw.slice(0, 4096)}...` : raw; + } + + return new Promise((resolve, reject) => { + const request = http.request({ + hostname: url.hostname, + port: url.port || 80, + path: `${url.pathname}${url.search}`, + method: 'POST', + headers: { + 'content-type': 'application/json', + 'content-length': body.length, + 'x-openshell-mcp-conformance-token': bridgeToken, + }, + agent: false, + }, (response) => { + const chunks = []; + response.on('data', (chunk) => chunks.push(chunk)); + response.on('end', () => { + const raw = Buffer.concat(chunks).toString('utf8'); + const statusCode = response.statusCode ?? 0; + try { + const parsed = JSON.parse(raw); + if (statusCode < 200 || statusCode >= 300) { + reject(new Error(`bridge callback failed (HTTP ${statusCode}): ${snippet(raw)}`)); + return; + } + resolve(parsed); + } catch (error) { + reject(new Error(`bridge returned invalid JSON (HTTP ${statusCode}): ${snippet(raw) || error.message}`)); + } + }); + }); + request.on('error', reject); + request.setTimeout(timeoutMs, () => { + request.destroy(new Error(`bridge callback timed out after ${timeoutMs}ms to ${url.toString()}`)); + }); + request.end(body); + }); +} + +const forwardedEnv = {}; +for (const name of [ + 'MCP_CONFORMANCE_SCENARIO', + 'MCP_CONFORMANCE_CONTEXT', + 'MCP_CONFORMANCE_PROTOCOL_VERSION', +]) { + if (process.env[name] !== undefined) { + forwardedEnv[name] = process.env[name]; + } +} + +const body = await postJson(parsedBridgeUrl, { + server_url: rewriteServerUrl(serverUrl), + env: forwardedEnv, +}); +if (typeof body.exit_code !== 'number') { + const raw = JSON.stringify(body); + throw new Error(`bridge returned JSON without numeric exit_code: ${raw.length > 4096 ? `${raw.slice(0, 4096)}...` : raw}`); +} +if (body.stdout) { + process.stdout.write(body.stdout); +} +if (body.stderr) { + process.stderr.write(body.stderr); +} +process.exit(body.exit_code); diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 083c622df4..2f61f2d868 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -97,6 +97,11 @@ name = "forward_proxy_graphql_l7" path = "tests/forward_proxy_graphql_l7.rs" required-features = ["e2e-host-gateway"] +[[test]] +name = "forward_proxy_jsonrpc_l7" +path = "tests/forward_proxy_jsonrpc_l7.rs" +required-features = ["e2e-host-gateway"] + [[test]] name = "gpu_device_selection" path = "tests/gpu_device_selection.rs" diff --git a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs new file mode 100644 index 0000000000..d5196b3554 --- /dev/null +++ b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs @@ -0,0 +1,520 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! E2E tests for JSON-RPC L7 inspection across both proxy entry points. +//! +//! The upstream server deliberately does not implement JSON-RPC. `OpenShell` +//! parses and enforces JSON-RPC before forwarding, so any HTTP server that +//! accepts POST /rpc is enough to prove allowed requests reach upstream +//! and denied requests are stopped by the sandbox proxy. + +#![cfg(feature = "e2e")] + +use std::io::Write; + +use openshell_e2e::harness::container::ContainerHttpServer; +use openshell_e2e::harness::sandbox::SandboxGuard; +use tempfile::NamedTempFile; + +const RULES_TEST_SERVER_ALIAS: &str = "jsonrpc-l7-rules.openshell.test"; +const AUDIT_TEST_SERVER_ALIAS: &str = "jsonrpc-l7-audit.openshell.test"; + +async fn start_test_server(alias: &str) -> Result { + let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer + +class Handler(BaseHTTPRequestHandler): + def read_body(self): + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + data = b"" + while True: + size_line = self.rfile.readline() + if not size_line: + break + size = int(size_line.split(b";", 1)[0].strip(), 16) + if size == 0: + while self.rfile.readline().strip(): + pass + break + data += self.rfile.read(size) + self.rfile.read(2) + return data + return self.rfile.read(int(self.headers.get("Content-Length", "0"))) + + def do_GET(self): + self.send_response(200) + self.end_headers() + + def do_POST(self): + self.read_body() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b'{"jsonrpc":"2.0","id":1,"result":{}}') + + def log_message(self, format, *args): + pass + +HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() +"#; + + ContainerHttpServer::start_python(alias, script).await +} + +fn write_jsonrpc_policy(host: &str, port: u16) -> Result { + let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + test_jsonrpc_l7: + name: test_jsonrpc_l7 + endpoints: + - host: {host} + port: {port} + path: /rpc + protocol: json-rpc + enforcement: enforce + allowed_ips: + - "10.0.0.0/8" + - "172.0.0.0/8" + - "192.168.0.0/16" + - "fc00::/7" + json_rpc: + max_body_bytes: 65536 + rules: + - allow: + method: initialize + - allow: + method: tools/list + - allow: + method: tools/call + deny_rules: + - method: tools/delete + binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|e| format!("write temp policy file: {e}"))?; + file.flush() + .map_err(|e| format!("flush temp policy file: {e}"))?; + Ok(file) +} + +fn write_jsonrpc_default_audit_policy(host: &str, port: u16) -> Result { + let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + test_jsonrpc_l7_audit: + name: test_jsonrpc_l7_audit + endpoints: + - host: {host} + port: {port} + path: /rpc + protocol: json-rpc + allowed_ips: + - "10.0.0.0/8" + - "100.64.0.0/10" + - "172.0.0.0/8" + - "198.18.0.0/15" + - "192.168.0.0/16" + - "fc00::/7" + json_rpc: + max_body_bytes: 65536 + rules: + - allow: + method: initialize + binaries: + - path: /usr/bin/python* + - path: /usr/local/bin/python* + - path: /sandbox/.uv/python/*/bin/python* +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|e| format!("write temp policy file: {e}"))?; + file.flush() + .map_err(|e| format!("flush temp policy file: {e}"))?; + Ok(file) +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn jsonrpc_l7_enforces_method_rules_on_forward_and_connect_paths() { + let server = start_test_server(RULES_TEST_SERVER_ALIAS) + .await + .expect("start test server"); + let policy = write_jsonrpc_policy(&server.host, server.port).expect("write custom policy"); + let policy_path = policy + .path() + .to_str() + .expect("temp policy path should be utf-8") + .to_string(); + + let script = format!( + r#" +import json +import os +import socket +import time +import urllib.error +import urllib.parse +import urllib.request + +HOST = {host:?} +PORT = {port} +DETAILS = {{ + "debug_target": {{"host": HOST, "port": PORT}}, + "debug_proxy_env": {{ + "http_proxy": os.environ.get("http_proxy"), + "https_proxy": os.environ.get("https_proxy"), + "HTTP_PROXY": os.environ.get("HTTP_PROXY"), + "HTTPS_PROXY": os.environ.get("HTTPS_PROXY"), + "NO_PROXY": os.environ.get("NO_PROXY"), + "no_proxy": os.environ.get("no_proxy"), + }}, +}} + +def text(data): + return data.decode(errors="replace") + +def selected_headers(headers): + return {{ + key.lower(): value + for key, value in headers.items() + if key.lower() in ("content-type", "content-length", "server") + }} + +def record_http_error(label, error, request_body): + response_body = error.read() + DETAILS[f"{{label}}_request"] = request_body + DETAILS[f"{{label}}_response"] = {{ + "status": error.code, + "reason": str(error.reason), + "headers": selected_headers(error.headers), + "body": text(response_body), + }} + return error.code + +def post_jsonrpc(label, method, params=None, req_id=1): + body = {{"jsonrpc": "2.0", "id": req_id, "method": method}} + if params is not None: + body["params"] = params + encoded = json.dumps(body).encode() + request = urllib.request.Request( + f"http://{{HOST}}:{{PORT}}/rpc", + data=encoded, + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, body) + +def post_jsonrpc_batch(label, requests): + encoded = json.dumps(requests).encode() + request = urllib.request.Request( + f"http://{{HOST}}:{{PORT}}/rpc", + data=encoded, + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, requests) + +def post_invalid_json(label): + encoded = b"not valid json {{" + request = urllib.request.Request( + f"http://{{HOST}}:{{PORT}}/rpc", + data=encoded, + headers={{"Content-Type": "application/json", "Content-Length": str(len(encoded))}}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, text(encoded)) + +def proxy_parts(*names): + proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_until(sock, marker): + data = b"" + while marker not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def read_response(sock): + response = read_until(sock, b"\r\n\r\n") + headers, _, body = response.partition(b"\r\n\r\n") + content_length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + content_length = int(line.split(b":", 1)[1].strip()) + break + while len(body) < content_length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + return response, body + +def status_code(response, label): + parts = response.split() + if len(parts) < 2: + DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{{label}}: malformed HTTP response: {{response!r}}") + try: + return int(parts[1]) + except ValueError as error: + DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error + +def record_raw_response(label, response, body=b""): + code = status_code(response, label) + if code != 200: + DETAILS[f"{{label}}_raw"] = text(response) + if body: + DETAILS[f"{{label}}_body"] = text(body) + return code + +def connect_http_status(label, request): + proxy_host, proxy_port = proxy_parts("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + target = f"{{HOST}}:{{PORT}}" + + last_error = None + for attempt in range(5): + try: + with socket.create_connection((proxy_host, proxy_port), timeout=15) as sock: + sock.sendall( + f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode() + ) + connect_response = read_until(sock, b"\r\n\r\n") + connect_code = record_raw_response(f"{{label}}_connect", connect_response) + if connect_code != 200: + return connect_code + sock.sendall(request) + sock.shutdown(socket.SHUT_WR) + response, body = read_response(sock) + return record_raw_response(f"{{label}}_response", response, body) + except (OSError, RuntimeError) as error: + last_error = error + DETAILS[f"{{label}}_attempt_{{attempt + 1}}_error"] = str(error) + time.sleep(0.2) + + raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") + +def connect_jsonrpc_status(method, params, label): + target = f"{{HOST}}:{{PORT}}" + body = {{"jsonrpc": "2.0", "id": 1, "method": method}} + if params is not None: + body["params"] = params + encoded = json.dumps(body).encode() + request = ( + f"POST /rpc HTTP/1.1\r\n" + f"Host: {{target}}\r\n" + f"Content-Type: application/json\r\n" + f"Content-Length: {{len(encoded)}}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + encoded + return connect_http_status(label, request) + +results = {{ + # forward proxy — method-only allow rules + "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}), + "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), + + # forward proxy — method allow/deny rules + "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {{"name": "read_status"}}), + "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}), + "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {{"name": "purge_cache"}}), + + # forward proxy — batch: all requests allowed + "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ + {{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}}, + {{"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {{"name": "read_status"}}}}, + ]), + + # forward proxy — batch: one denied request causes full batch denial + "forward_batch_one_denied": post_jsonrpc_batch("forward_batch_one_denied", [ + {{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}}, + {{"jsonrpc": "2.0", "id": 2, "method": "tools/delete", "params": {{"name": "purge_cache"}}}}, + ]), + + # forward proxy — invalid JSON body fails closed before generic rules apply + "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), + + # CONNECT path — representative allowed and denied cases + "connect_method_initialize_allowed": connect_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "connect_method_initialize_allowed"), + "connect_method_tools_list_allowed": connect_jsonrpc_status("tools/list", None, "connect_method_tools_list_allowed"), + "connect_method_tools_call_allowed": connect_jsonrpc_status("tools/call", {{"name": "read_status"}}, "connect_method_tools_call_allowed"), + "connect_method_tools_call_with_unmatched_params_allowed": connect_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "connect_method_tools_call_with_unmatched_params_allowed"), + "connect_method_tools_delete_denied": connect_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "connect_method_tools_delete_denied"), +}} +results.update(DETAILS) +print(json.dumps(results, sort_keys=True)) +"#, + host = server.host, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + + for (key, expected) in [ + // forward proxy — allowed + ("forward_method_initialize_allowed", 200), + ("forward_method_tools_list_allowed", 200), + ("forward_method_tools_call_allowed", 200), + ("forward_method_tools_call_with_unmatched_params_allowed", 200), + // forward proxy — method denied + ("forward_method_tools_delete_denied", 403), + // forward proxy — batch + ("forward_batch_all_allowed", 200), + ("forward_batch_one_denied", 403), + // forward proxy — parse error + ("forward_invalid_json_denied", 403), + // CONNECT path — allowed + ("connect_method_initialize_allowed", 200), + ("connect_method_tools_list_allowed", 200), + ("connect_method_tools_call_allowed", 200), + ("connect_method_tools_call_with_unmatched_params_allowed", 200), + // CONNECT path — method denied + ("connect_method_tools_delete_denied", 403), + ] { + let expected_fragment = format!(r#""{key}": {expected}"#); + assert!( + guard.create_output.contains(&expected_fragment), + "expected {key}={expected}, got:\n{}", + guard.create_output + ); + } +} + +#[tokio::test] +async fn jsonrpc_forward_proxy_hard_denies_response_frames_in_default_audit_mode() { + let server = start_test_server(AUDIT_TEST_SERVER_ALIAS) + .await + .expect("start test server"); + let policy = + write_jsonrpc_default_audit_policy(&server.host, server.port).expect("write custom policy"); + let policy_path = policy + .path() + .to_str() + .expect("temp policy path should be utf-8") + .to_string(); + + let script = format!( + r#" +import json +import urllib.error +import urllib.request + +HOST = {host:?} +PORT = {port} + +def post_jsonrpc(body): + encoded = json.dumps(body).encode() + request = urllib.request.Request( + f"http://{{HOST}}:{{PORT}}/rpc", + data=encoded, + headers={{"Content-Type": "application/json"}}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +results = {{ + "forward_unknown_method_audited": post_jsonrpc({{"jsonrpc": "2.0", "id": 1, "method": "unknown/method"}}), + "forward_response_frame_hard_denied": post_jsonrpc({{"jsonrpc": "2.0", "id": 1, "result": {{}}}}), +}} +print(json.dumps(results, sort_keys=True)) +"#, + host = server.host, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + + for (key, expected) in [ + ("forward_unknown_method_audited", 200), + ("forward_response_frame_hard_denied", 403), + ] { + let expected_fragment = format!(r#""{key}": {expected}"#); + assert!( + guard.create_output.contains(&expected_fragment), + "expected {key}={expected}, got:\n{}", + guard.create_output + ); + } +} diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index fe9156f980..64062b74d6 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -21,7 +21,7 @@ # The default community sandbox image uses :latest. This wrapper refreshes it # before starting the gateway, while the Docker driver defaults to IfNotPresent # so local Dockerfile-built images remain usable. - +# set -euo pipefail if [ "$#" -eq 0 ]; then diff --git a/mise.toml b/mise.toml index 04e040421e..fc5ba340aa 100644 --- a/mise.toml +++ b/mise.toml @@ -65,7 +65,7 @@ DOCKER_BUILDKIT = "1" [vars] # Python paths to include in formatting/linting -python_paths = "python/ tasks/scripts/*.py deploy/sbom/*.py" +python_paths = "python/ tasks/scripts/*.py e2e/mcp-conformance/*.py deploy/sbom/*.py" [task_config] includes = ["tasks/*.toml"] diff --git a/proto/sandbox.proto b/proto/sandbox.proto index faf7569acf..8a5a593334 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -138,6 +138,43 @@ message NetworkEndpoint { // AWS region override for SigV4 signing. When set, takes precedence over // hostname-based region extraction. Required for non-standard endpoints. string signing_region = 21; + // Maximum JSON-RPC-over-HTTP request body bytes to buffer for inspection. + // Defaults to 65536 when unset. + uint32 json_rpc_max_body_bytes = 22; + // MCP-only policy and inspection options. Only used when protocol is "mcp". + McpOptions mcp = 23; +} + +// MCP options are grouped so MCP-specific policy can grow without adding more +// top-level NetworkEndpoint fields. Current enforcement targets the active +// 2025-11-25 Streamable HTTP/tools behavior, while preserving space for +// version-profile policy if OpenShell adopts 2026-07-28 draft behavior later. +// +// Planned policy extensions should use OpenShell-owned static definitions for +// MCP method/version profiles rather than treating dependency enums as the +// policy contract. Candidate profile checks include request metadata/header +// validation, response/SSE introspection, trusted annotation handling, +// resultType/cache metadata validation, x-mcp-header tool-definition checks, +// and subscriptions/listen handling. +// +// Sources: +// - https://modelcontextprotocol.io/specification/2025-11-25/server/tools +// - https://modelcontextprotocol.io/specification/draft/changelog +// - https://modelcontextprotocol.io/specification/draft/basic/transports/streamable-http +// - https://modelcontextprotocol.io/specification/draft/server/tools +message McpOptions { + // Hardening boundary for tools/call params.name. When unset or true, the + // supervisor enforces the MCP recommended tool-name syntax + // ^[A-Za-z0-9_.-]{1,128}$ before policy evaluation. Set false only for + // compatibility with servers that intentionally use non-recommended names. + // + // Source: + // - https://modelcontextprotocol.io/specification/2025-11-25/server/tools#tool-names + optional bool strict_tool_names = 1; + // Method-layer default for MCP endpoints. When true, OpenShell allows parsed + // MCP-family methods at the method layer unless a tool-name policy narrows + // tools/call. When unset or false, explicit method rules are required. + optional bool allow_all_known_mcp_methods = 2; } // Trusted GraphQL operation classification. @@ -154,7 +191,8 @@ message GraphqlOperation { // Mirrors L7Allow — same fields, same matching semantics, inverted effect. // Deny rules are evaluated after allow rules and take precedence. message L7DenyRule { - // HTTP method (REST): GET, POST, etc. or "*" for any. + // Protocol method: HTTP method (REST/WebSocket), JSON-RPC method name, or + // "*" for any when supported by the protocol. string method = 1; // URL path glob pattern (REST): "/repos/*/pulls/*/reviews", "**" for any. string path = 2; @@ -170,6 +208,10 @@ message L7DenyRule { // GraphQL root field globs. Deny rules match when any selected root field // matches any configured glob. repeated string fields = 7; + reserved 8; + // MCP params matcher map. Currently only params.name is supported for + // tools/call filtering. Generic protocol "json-rpc" rejects params matchers. + map params = 9; } // An L7 policy rule (allow-only). @@ -179,7 +221,8 @@ message L7Rule { // Allowed action definition for L7 rules. message L7Allow { - // HTTP method (REST): GET, POST, etc. or "*" for any. + // Protocol method: HTTP method (REST/WebSocket), JSON-RPC method name, or + // "*" for any when supported by the protocol. string method = 1; // URL path glob pattern (REST): "/repos/**", "**" for any. string path = 2; @@ -196,6 +239,10 @@ message L7Allow { // GraphQL root field globs. Allow rules match only when every selected root // field matches one of the configured globs. Omit to match all fields. repeated string fields = 7; + reserved 8; + // MCP params matcher map. Currently only params.name is supported for + // tools/call filtering. Generic protocol "json-rpc" rejects params matchers. + map params = 9; } // Query value matcher for one query parameter key. diff --git a/tasks/test.toml b/tasks/test.toml index 1d0f97856a..a3c56bb2bb 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -29,8 +29,8 @@ run = "tasks/scripts/test-packaging-assets.sh" hide = true [e2e] -description = "Run all end-to-end tests (Rust + Python)" -depends = ["e2e:rust", "e2e:python"] +description = "Run all end-to-end tests (Rust + Python + MCP)" +depends = ["e2e:rust", "e2e:python", "e2e:mcp"] ["e2e:gpu"] description = "Run Docker GPU end-to-end tests" @@ -71,6 +71,14 @@ run = [ "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test websocket_conformance", ] +["e2e:mcp"] +description = "Run MCP conformance e2e scenarios against one Docker-backed gateway (static defaults for spec 2025-11-25; set OPENSHELL_MCP_CONFORMANCE_SCENARIOS for a focused subset)" +run = "bash e2e/mcp-conformance.sh" + +["e2e:nodejs"] +description = "Alias for e2e:mcp" +depends = ["e2e:mcp"] + ["e2e:python"] description = "Run Python e2e tests against a Docker-backed gateway (E2E_PARALLEL=N or 'auto'; default 5)" depends = ["python:proto"] From d28754281f6dbf5d57ab24e21358301eb6c765fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20Hu=C3=9F?= Date: Wed, 24 Jun 2026 15:48:49 +0200 Subject: [PATCH 20/20] feat(cli): add --output json/yaml to sandbox get, status, and sandbox create MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add structured output support to three commands that previously only supported human-readable table output: - `sandbox get --output json/yaml`: emits sandbox detail including policy source, revision, and active policy content via a new `sandbox_detail_to_json` converter. - `status --output json/yaml`: emits gateway connection state via a new `status_to_json` converter with conditional fields (auth, version, error, http_status) matching the human output. - `sandbox create --output json/yaml`: emits sandbox metadata after the sandbox reaches Ready phase, suppressing spinners and ANSI chrome from stdout. Uploads and port forwarding still execute before output. Clap rejects `--output` combined with `--editor` or trailing commands. All three commands reuse the existing `OutputFormat` enum and `print_output_single` helper. Default output (no --output flag) is byte-identical to current behavior. Unit tests cover both converters. Closes #1964 Assisted-By: 🤖 Claude Code Signed-off-by: Roland Huß --- crates/openshell-cli/src/main.rs | 39 +- crates/openshell-cli/src/run.rs | 372 +++++++++++++++--- .../sandbox_create_lifecycle_integration.rs | 12 + .../sandbox_name_fallback_integration.rs | 8 +- docs/sandboxes/manage-gateways.mdx | 6 + docs/sandboxes/manage-sandboxes.mdx | 12 + 6 files changed, 382 insertions(+), 67 deletions(-) diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index e02c7c9cd5..a46870b072 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -533,7 +533,11 @@ enum Commands { /// Show gateway status and information. #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] - Status, + Status { + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, + }, /// Manage inference configuration. #[command(after_help = INFERENCE_EXAMPLES, help_template = SUBCOMMAND_HELP_TEMPLATE)] @@ -1336,6 +1340,10 @@ enum SandboxCommands { #[arg(long, value_parser = ["manual", "auto"], default_value = "manual")] approval_mode: String, + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table, conflicts_with_all = ["editor", "command", "no_keep"])] + output: OutputFormat, + /// Command to run after "--" (defaults to an interactive shell). #[arg(last = true, allow_hyphen_values = true)] command: Vec, @@ -1349,8 +1357,12 @@ enum SandboxCommands { name: Option, /// Print only the active policy YAML (same policy as the default view; stdout only). - #[arg(long)] + #[arg(long, conflicts_with = "output")] policy_only: bool, + + /// Output format. + #[arg(short = 'o', long = "output", value_enum, default_value_t = OutputFormat::Table)] + output: OutputFormat, }, /// List sandboxes. @@ -2083,11 +2095,17 @@ async fn main() -> Result<()> { // ----------------------------------------------------------- // Top-level status // ----------------------------------------------------------- - Some(Commands::Status) => { + Some(Commands::Status { output }) => { if let Ok(ctx) = resolve_gateway(&cli.gateway, &cli.gateway_endpoint) { let mut tls = tls.with_gateway_name(&ctx.name); apply_auth(&mut tls, &ctx.name); - run::gateway_status(&ctx.name, &ctx.endpoint, &tls).await?; + run::gateway_status(&ctx.name, &ctx.endpoint, output.as_str(), &tls).await?; + } else if openshell_cli::output::print_output_single( + output.as_str(), + &(), + |()| serde_json::json!({"status": "not_configured"}), + )? { + // Structured output handled. } else { println!("{}", "Gateway Status".cyan().bold()); println!(); @@ -2618,6 +2636,7 @@ async fn main() -> Result<()> { labels, envs, approval_mode, + output, command, } => { // Resolve --tty / --no-tty into an Option override. @@ -2704,6 +2723,7 @@ async fn main() -> Result<()> { labels: labels_map, environment: env_map, approval_mode: &approval_mode, + output: output.as_str(), }, &tls, )) @@ -2777,9 +2797,14 @@ async fn main() -> Result<()> { | SandboxCommands::Download { .. } => { unreachable!() } - SandboxCommands::Get { name, policy_only } => { + SandboxCommands::Get { + name, + policy_only, + output, + } => { let name = resolve_sandbox_name(name, &ctx.name)?; - run::sandbox_get(endpoint, &name, policy_only, &tls).await?; + run::sandbox_get(endpoint, &name, policy_only, output.as_str(), &tls) + .await?; } SandboxCommands::List { limit, @@ -3463,7 +3488,7 @@ mod tests { .expect("global gateway flag should parse with subcommands"); assert_eq!(cli.gateway.as_deref(), Some("demo")); - assert!(matches!(cli.command, Some(Commands::Status))); + assert!(matches!(cli.command, Some(Commands::Status { .. }))); } #[test] diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index f56bb71512..dd873eff0c 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -41,7 +41,7 @@ use openshell_core::proto::{ DeleteServiceRequest, DetachSandboxProviderRequest, ExecSandboxRequest, ExposeServiceRequest, GetClusterInferenceRequest, GetDraftHistoryRequest, GetDraftPolicyRequest, GetGatewayConfigRequest, GetProviderProfileRequest, GetProviderRefreshStatusRequest, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxLogsRequest, + GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, GetSandboxLogsRequest, GetSandboxPolicyStatusRequest, GetSandboxRequest, GetServiceRequest, GpuResourceRequirements, HealthRequest, ImportProviderProfilesRequest, LintProviderProfilesRequest, ListProviderProfilesRequest, ListProvidersRequest, ListSandboxPoliciesRequest, @@ -527,53 +527,105 @@ fn print_sandbox_header(sandbox: &Sandbox, display: Option<&ProvisioningDisplay> /// Show gateway status. #[allow(clippy::branches_sharing_code)] -pub async fn gateway_status(gateway_name: &str, server: &str, tls: &TlsOptions) -> Result<()> { - println!("{}", "Server Status".cyan().bold()); - println!(); - println!(" {} {}", "Gateway:".dimmed(), gateway_name); - println!(" {} {}", "Server:".dimmed(), server); - if tls.is_bearer_auth() { - println!(" {} Edge (bearer token)", "Auth:".dimmed()); - } - - // Try to connect and get health - match grpc_client(server, tls).await { +pub async fn gateway_status( + gateway_name: &str, + server: &str, + output: &str, + tls: &TlsOptions, +) -> Result<()> { + let is_bearer = tls.is_bearer_auth(); + + // Build status data before any output. + let (status_str, version, error, http_status): ( + &str, + Option, + Option, + Option, + ) = match grpc_client(server, tls).await { Ok(mut client) => match client.health(HealthRequest {}).await { Ok(response) => { let health = response.into_inner(); - println!(" {} {}", "Status:".dimmed(), "Connected".green()); - println!(" {} {}", "Version:".dimmed(), health.version); + ("connected", Some(health.version), None, None) } - Err(e) => { - if let Some(status) = http_health_check(server, tls).await? { - if status.is_success() { - println!(" {} {}", "Status:".dimmed(), "Connected (HTTP)".yellow()); - println!(" {} {}", "HTTP: ".dimmed(), status); - println!(" {} {}", "gRPC error:".dimmed(), e); + Err(e) => http_health_check(server, tls).await?.map_or_else( + || ("error", None, Some(e.to_string()), None), + |http| { + let hs = Some(http.to_string()); + if http.is_success() { + ("connected_http", None, Some(e.to_string()), hs) } else { - println!(" {} {}", "Status:".dimmed(), "Error".red()); - println!(" {} {}", "HTTP:".dimmed(), status); - println!(" {} {}", "gRPC error:".dimmed(), e); + ("error", None, Some(e.to_string()), hs) } + }, + ), + }, + Err(e) => http_health_check(server, tls).await?.map_or_else( + || ("disconnected", None, Some(e.to_string()), None), + |http| { + let hs = Some(http.to_string()); + if http.is_success() { + ("connected_http", None, Some(e.to_string()), hs) } else { - println!(" {} {}", "Status:".dimmed(), "Error".red()); - println!(" {} {}", "Error:".dimmed(), e); + ("disconnected", None, Some(e.to_string()), hs) } + }, + ), + }; + + let json_data = status_to_json( + gateway_name, + server, + is_bearer, + status_str, + &version, + &error, + &http_status, + ); + if crate::output::print_output_single(output, &json_data, Clone::clone)? { + return Ok(()); + } + + // Human-readable output (existing behavior). + println!("{}", "Server Status".cyan().bold()); + println!(); + println!(" {} {}", "Gateway:".dimmed(), gateway_name); + println!(" {} {}", "Server:".dimmed(), server); + if is_bearer { + println!(" {} Edge (bearer token)", "Auth:".dimmed()); + } + match status_str { + "connected" => { + println!(" {} {}", "Status:".dimmed(), "Connected".green()); + if let Some(ref v) = version { + println!(" {} {}", "Version:".dimmed(), v); } - }, - Err(e) => { - if let Some(status) = http_health_check(server, tls).await? { - if status.is_success() { - println!(" {} {}", "Status:".dimmed(), "Connected (HTTP)".yellow()); - println!(" {} {}", "HTTP:".dimmed(), status); + } + "connected_http" => { + println!(" {} {}", "Status:".dimmed(), "Connected (HTTP)".yellow()); + if let Some(ref hs) = http_status { + println!(" {} {}", "HTTP: ".dimmed(), hs); + } + if let Some(ref e) = error { + println!(" {} {}", "gRPC error:".dimmed(), e); + } + } + "error" => { + println!(" {} {}", "Status:".dimmed(), "Error".red()); + if let Some(ref hs) = http_status { + println!(" {} {}", "HTTP:".dimmed(), hs); + if let Some(ref e) = error { println!(" {} {}", "gRPC error:".dimmed(), e); - } else { - println!(" {} {}", "Status:".dimmed(), "Disconnected".red()); - println!(" {} {}", "HTTP:".dimmed(), status); - println!(" {} {}", "Error:".dimmed(), e); } - } else { - println!(" {} {}", "Status:".dimmed(), "Disconnected".red()); + } else if let Some(ref e) = error { + println!(" {} {}", "Error:".dimmed(), e); + } + } + _ => { + println!(" {} {}", "Status:".dimmed(), "Disconnected".red()); + if let Some(ref hs) = http_status { + println!(" {} {}", "HTTP:".dimmed(), hs); + } + if let Some(ref e) = error { println!(" {} {}", "Error:".dimmed(), e); } } @@ -582,6 +634,36 @@ pub async fn gateway_status(gateway_name: &str, server: &str, tls: &TlsOptions) Ok(()) } +fn status_to_json( + gateway_name: &str, + server: &str, + is_bearer: bool, + status: &str, + version: &Option, + error: &Option, + http_status: &Option, +) -> serde_json::Value { + let mut obj = serde_json::json!({ + "gateway": gateway_name, + "server": server, + "status": status, + }); + let map = obj.as_object_mut().expect("json! returns object"); + if is_bearer { + map.insert("auth".into(), serde_json::json!("edge_bearer")); + } + if let Some(v) = version { + map.insert("version".into(), serde_json::json!(v)); + } + if let Some(hs) = http_status { + map.insert("http_status".into(), serde_json::json!(hs)); + } + if let Some(e) = error { + map.insert("error".into(), serde_json::json!(e)); + } + obj +} + /// Set the active gateway. pub fn gateway_use(name: &str) -> Result<()> { // Verify the gateway exists @@ -1789,6 +1871,7 @@ pub struct SandboxCreateConfig<'a> { pub labels: HashMap, pub environment: HashMap, pub approval_mode: &'a str, + pub output: &'a str, } impl Default for SandboxCreateConfig<'_> { @@ -1812,6 +1895,7 @@ impl Default for SandboxCreateConfig<'_> { labels: HashMap::new(), environment: HashMap::new(), approval_mode: "manual", + output: "text", } } } @@ -1842,6 +1926,7 @@ pub async fn sandbox_create( labels, environment, approval_mode, + output, } = config; if editor.is_some() && !command.is_empty() { @@ -1991,23 +2076,30 @@ pub async fn sandbox_create( } } + let structured_output = output != "table"; + // Set up display — interactive terminals get a step-based checklist with // spinners; non-interactive (pipes / CI) get timestamped lines. - let mut display = if interactive { + // Suppress all human chrome when structured output is active. + let mut display = if interactive && !structured_output { Some(ProvisioningDisplay::new()) } else { None }; - // Print header - print_sandbox_header(&sandbox, display.as_ref()); - - // Set initial active step on the spinner. - if let Some(d) = display.as_mut() { - d.set_active_step(ProvisioningStep::RequestingSandbox); + if structured_output { + eprintln!("Provisioning sandbox (structured output on stdout)..."); } else { - let ts = format_timestamp(Duration::ZERO); - println!(" {} Requesting compute...", ts.dimmed()); + // Print header + print_sandbox_header(&sandbox, display.as_ref()); + + // Set initial active step on the spinner. + if let Some(d) = display.as_mut() { + d.set_active_step(ProvisioningStep::RequestingSandbox); + } else { + let ts = format_timestamp(Duration::ZERO); + println!(" {} Requesting compute...", ts.dimmed()); + } } // Non-interactive mode: track start time for timestamps. @@ -2041,6 +2133,7 @@ pub async fn sandbox_create( .into_inner(); let mut last_phase = sandbox.phase(); + let mut last_sandbox = sandbox.clone(); let mut last_error_reason = String::new(); let mut last_condition_message = ready_false_condition_message(sandbox.status.as_ref()); // Track whether we have seen a non-Ready phase during the watch. @@ -2070,7 +2163,9 @@ pub async fn sandbox_create( if let Some(d) = display.as_mut() { d.finish_error(&timeout_message); } - println!(); + if !structured_output { + println!(); + } return Err(miette::miette!(timeout_message)); } @@ -2089,7 +2184,9 @@ pub async fn sandbox_create( if let Some(d) = display.as_mut() { d.finish_error(&timeout_message); } - println!(); + if !structured_output { + println!(); + } return Err(miette::miette!(timeout_message)); } }; @@ -2099,6 +2196,7 @@ pub async fn sandbox_create( Some(openshell_core::proto::sandbox_stream_event::Payload::Sandbox(s)) => { let phase = SandboxPhase::try_from(s.phase()).unwrap_or(SandboxPhase::Unknown); last_phase = s.phase(); + last_sandbox = s.clone(); if let Some(message) = ready_false_condition_message(s.status.as_ref()) { last_condition_message = Some(message); } @@ -2281,6 +2379,11 @@ pub async fn sandbox_create( ); } + if structured_output { + crate::output::print_output_single(output, &last_sandbox, sandbox_to_json)?; + return Ok(()); + } + if let Some(editor) = editor { let ssh_gateway_name = effective_tls.gateway_name().unwrap_or(gateway_name); sandbox_connect_editor( @@ -2611,6 +2714,7 @@ pub async fn sandbox_get( server: &str, name: &str, policy_only: bool, + output: &str, tls: &TlsOptions, ) -> Result<()> { let mut client = grpc_client(server, tls).await?; @@ -2650,6 +2754,11 @@ pub async fn sandbox_get( return Ok(()); } + let detail_json = sandbox_detail_to_json(&sandbox, &config)?; + if crate::output::print_output_single(output, &detail_json, Clone::clone)? { + return Ok(()); + } + println!("{}", "Sandbox:".cyan().bold()); println!(); let id = if sandbox.object_id().is_empty() { @@ -3438,6 +3547,48 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { }) } +fn sandbox_detail_to_json( + sandbox: &Sandbox, + config: &GetSandboxConfigResponse, +) -> Result { + let mut value = sandbox_to_json(sandbox); + let obj = value + .as_object_mut() + .expect("sandbox_to_json returns object"); + + let policy_source = if config.policy_source == PolicySource::Global as i32 { + "global" + } else { + "sandbox" + }; + obj.insert("policy_source".into(), serde_json::json!(policy_source)); + + let policy_from_global = config.policy_source == PolicySource::Global as i32; + let revision = if policy_from_global { + if config.global_policy_version > 0 { + Some(config.global_policy_version) + } else if config.version > 0 { + Some(config.version) + } else { + None + } + } else if config.version > 0 { + Some(config.version) + } else { + None + }; + obj.insert("revision".into(), serde_json::json!(revision)); + + let policy_json = match config.policy.as_ref() { + Some(p) => openshell_policy::sandbox_policy_to_json_value(p) + .wrap_err("failed to convert policy to JSON")?, + None => serde_json::Value::Null, + }; + obj.insert("policy".into(), policy_json); + + Ok(value) +} + pub async fn sandbox_provider_list(server: &str, name: &str, tls: &TlsOptions) -> Result<()> { let mut client = grpc_client(server, tls).await?; let response = client @@ -6432,10 +6583,7 @@ pub async fn gateway_settings_get(server: &str, json: bool, tls: &TlsOptions) -> Ok(()) } -fn settings_to_json_sandbox( - name: &str, - response: &openshell_core::proto::GetSandboxConfigResponse, -) -> serde_json::Value { +fn settings_to_json_sandbox(name: &str, response: &GetSandboxConfigResponse) -> serde_json::Value { let policy_source = if response.policy_source == PolicySource::Global as i32 { "global" } else { @@ -7870,10 +8018,11 @@ mod tests { PROGRESS_STEP_STARTING_SANDBOX, }; use openshell_core::proto::{ - GpuResourceRequirements, Provider, ProviderCredentialRefresh, - ProviderCredentialRefreshStatus, ProviderCredentialRefreshStrategy, - ProviderCredentialTokenGrant, ProviderProfile, ProviderProfileCredential, - ResourceRequirements, SandboxCondition, SandboxStatus, datamodel::v1::ObjectMeta, + GetSandboxConfigResponse, GpuResourceRequirements, PolicySource, Provider, + ProviderCredentialRefresh, ProviderCredentialRefreshStatus, + ProviderCredentialRefreshStrategy, ProviderCredentialTokenGrant, ProviderProfile, + ProviderProfileCredential, ResourceRequirements, Sandbox, SandboxCondition, SandboxPhase, + SandboxStatus, datamodel::v1::ObjectMeta, }; struct EnvVarGuard { @@ -9698,4 +9847,115 @@ mod tests { "raw milliseconds field should not exist" ); } + + #[test] + fn sandbox_detail_to_json_includes_policy_fields() { + let mut sandbox = Sandbox { + metadata: Some(ObjectMeta { + id: "sb-123".to_string(), + name: "test-sb".to_string(), + resource_version: 5, + created_at_ms: 1_609_459_200_000, + ..Default::default() + }), + ..Default::default() + }; + sandbox.set_phase(SandboxPhase::Ready as i32); + sandbox.set_current_policy_version(2); + + let config = GetSandboxConfigResponse { + policy_source: PolicySource::Global as i32, + global_policy_version: 3, + ..Default::default() + }; + + let json = super::sandbox_detail_to_json(&sandbox, &config).unwrap(); + + assert_eq!(json["id"], "sb-123"); + assert_eq!(json["name"], "test-sb"); + assert_eq!(json["phase"], "Ready"); + assert_eq!(json["policy_source"], "global"); + assert_eq!(json["revision"], 3); + assert!(json["policy"].is_null()); + } + + #[test] + fn sandbox_detail_to_json_sandbox_source_without_policy() { + let sandbox = Sandbox { + metadata: Some(ObjectMeta { + id: "sb-456".to_string(), + name: "no-policy-sb".to_string(), + ..Default::default() + }), + ..Default::default() + }; + let config = GetSandboxConfigResponse { + policy_source: PolicySource::Sandbox as i32, + version: 0, + ..Default::default() + }; + + let json = super::sandbox_detail_to_json(&sandbox, &config).unwrap(); + + assert_eq!(json["policy_source"], "sandbox"); + assert!(json["revision"].is_null()); + assert!(json["policy"].is_null()); + } + + #[test] + fn status_to_json_connected() { + let json = super::status_to_json( + "my-gw", + "http://127.0.0.1:8090", + false, + "connected", + &Some("1.2.3".to_string()), + &None, + &None, + ); + + assert_eq!(json["gateway"], "my-gw"); + assert_eq!(json["server"], "http://127.0.0.1:8090"); + assert_eq!(json["status"], "connected"); + assert_eq!(json["version"], "1.2.3"); + assert!(json.get("auth").is_none()); + assert!(json.get("error").is_none()); + assert!(json.get("http_status").is_none()); + } + + #[test] + fn status_to_json_disconnected_with_error() { + let json = super::status_to_json( + "broken-gw", + "http://10.0.0.1:8090", + false, + "disconnected", + &None, + &Some("connection refused".to_string()), + &None, + ); + + assert_eq!(json["status"], "disconnected"); + assert_eq!(json["error"], "connection refused"); + assert!(json.get("version").is_none()); + } + + #[test] + fn status_to_json_connected_http_with_bearer() { + let json = super::status_to_json( + "edge-gw", + "https://edge.example.com", + true, + "connected_http", + &None, + &Some("gRPC unavailable".to_string()), + &Some("200 OK".to_string()), + ); + + assert_eq!(json["status"], "connected_http"); + assert_eq!(json["auth"], "edge_bearer"); + assert_eq!(json["error"], "gRPC unavailable"); + assert_eq!(json["http_status"], "200 OK"); + assert!(json.get("version").is_none()); + } } diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index ec8bd53743..d1fc7c2616 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -1129,6 +1129,7 @@ async fn sandbox_create_keeps_command_sessions_by_default() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1161,6 +1162,7 @@ async fn sandbox_create_sends_cpu_and_memory_limits_only() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1228,6 +1230,7 @@ async fn sandbox_create_sends_driver_config_json() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1358,6 +1361,7 @@ async fn sandbox_create_does_not_infer_command_providers_when_v2_enabled() { tty_override: Some(true), ..test_config() }, + &tls, ) .await @@ -1403,6 +1407,7 @@ async fn sandbox_create_returns_vm_error_without_waiting_for_timeout() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1444,6 +1449,7 @@ async fn sandbox_create_keeps_waiting_while_vm_progress_arrives() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1477,6 +1483,7 @@ async fn sandbox_create_times_out_when_only_logs_arrive() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1507,6 +1514,7 @@ async fn sandbox_create_deletes_command_sessions_with_no_keep() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1541,6 +1549,7 @@ async fn sandbox_create_deletes_shell_sessions_with_no_keep() { tty_override: Some(true), ..test_config() }, + &tls, ) .await @@ -1574,6 +1583,7 @@ async fn sandbox_create_keeps_sandbox_with_hidden_keep_flag() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1609,6 +1619,7 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() { command: &["echo".into(), "OK".into()], ..test_config() }, + &tls, ) .await @@ -1748,6 +1759,7 @@ async fn sandbox_create_sends_environment_variables() { ]), ..test_config() }, + &tls, ) .await diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 8e799f821e..374298e1c3 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -628,7 +628,7 @@ async fn run_server() -> TestServer { async fn sandbox_get_sends_correct_name() { let ts = run_server().await; - run::sandbox_get(&ts.endpoint, "my-sandbox", false, &ts.tls) + run::sandbox_get(&ts.endpoint, "my-sandbox", false, "table", &ts.tls) .await .expect("sandbox_get should succeed"); @@ -645,7 +645,7 @@ async fn sandbox_get_sends_correct_name() { async fn sandbox_get_policy_only_round_trip() { let ts = run_server().await; - run::sandbox_get(&ts.endpoint, "my-sandbox", true, &ts.tls) + run::sandbox_get(&ts.endpoint, "my-sandbox", true, "table", &ts.tls) .await .expect("sandbox_get with policy_only should succeed"); @@ -671,7 +671,7 @@ async fn sandbox_get_with_persisted_last_sandbox() { assert_eq!(resolved, "persisted-sb"); // Call sandbox_get with the resolved name. - run::sandbox_get(&ts.endpoint, &resolved, false, &ts.tls) + run::sandbox_get(&ts.endpoint, &resolved, false, "table", &ts.tls) .await .expect("sandbox_get should succeed"); @@ -821,7 +821,7 @@ async fn explicit_name_takes_precedence_over_persisted() { // Persist one name, but supply a different one explicitly. save_last_sandbox("my-cluster", "old-sandbox").expect("save should succeed"); - run::sandbox_get(&ts.endpoint, "explicit-sandbox", false, &ts.tls) + run::sandbox_get(&ts.endpoint, "explicit-sandbox", false, "table", &ts.tls) .await .expect("sandbox_get should succeed"); diff --git a/docs/sandboxes/manage-gateways.mdx b/docs/sandboxes/manage-gateways.mdx index ce4496e8d2..e34a3cee91 100644 --- a/docs/sandboxes/manage-gateways.mdx +++ b/docs/sandboxes/manage-gateways.mdx @@ -110,6 +110,12 @@ Use `openshell status` for a quick health check: openshell status ``` +For automation or scripting, use `--output json` or `--output yaml` to get machine-readable output: + +```shell +openshell status --output json +``` + Use `openshell gateway info` when you need the registered endpoint, gateway metadata, or compute driver details: ```shell diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index fdbf497f5d..2b7f59a71b 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -20,6 +20,12 @@ Create a sandbox with a single command. For example, to create a sandbox with Cl openshell sandbox create -- claude ``` +For automation, use `--output json` or `--output yaml` to get machine-readable sandbox metadata after creation: + +```shell +openshell sandbox create --output json +``` + Every sandbox requires a gateway. Register or select one before running sandbox commands: ```shell @@ -294,6 +300,12 @@ Get detailed information about a specific sandbox. The output lists **Policy sou openshell sandbox get my-sandbox ``` +For automation, use `--output json` or `--output yaml` to get machine-readable sandbox details: + +```shell +openshell sandbox get my-sandbox --output json +``` + Print only that policy YAML for scripting (same effective policy, no metadata): ```shell