diff --git a/architecture/sandbox.md b/architecture/sandbox.md index d2cb44d7b8..5ebc29a69c 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -54,6 +54,21 @@ paths, such as proxy support files or GPU device paths when a GPU is present. All ordinary agent egress is routed through the sandbox proxy. The proxy identifies the calling binary, checks trust-on-first-use binary identity, rejects unsafe internal destinations, and evaluates the active policy. + +CONNECT and absolute-form forward HTTP are explicit-proxy adapters over the same +egress pipeline. Each adapter normalizes its request into an egress intent, and +the shared authorization result carries the process evidence used by destination +validation and relay selection. During the compatibility migration, endpoint +state is hydrated at the adapters' existing policy query points; it is not yet +one atomic, generation-consistent authorization result. Destination validation +returns an unopened connector so adapters retain their existing response and +upstream-dial timing. CONNECT prepares a generation-pinned relay context before +entering shared TLS-terminated or plaintext HTTP relays; non-HTTP traffic uses +the shared raw byte relay after the existing adapter gates. Forward HTTP retains +its guarded single-request relay while sharing authorization, request context, +policy-pinning, and destination boundaries. +Adapter-specific response and OCSF event shapes remain at the protocol boundary. + For inspected HTTP traffic, the proxy can enforce REST method/path rules, WebSocket upgrade and text-message rules, GraphQL operation rules, and MCP method, tool, and supported params rules or generic JSON-RPC method rules diff --git a/architecture/security-policy.md b/architecture/security-policy.md index a4d20b9b0f..89abccba0b 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -75,9 +75,9 @@ metadata before forwarding. The proxy also supports credential injection on terminated HTTP streams when policy allows the endpoint. Raw streams and long-lived response bodies are connection scoped. Policy -reloads affect the next connection or the next parsed HTTP request; they do not -rewrite bytes already being relayed. HTTP upgrades switch to raw relay by -default. A `protocol: rest` endpoint can opt in to +generation changes close relays pinned to the previous generation instead of +allowing them to continue under stale authorization. HTTP upgrades switch to +raw relay by default. A `protocol: rest` endpoint can opt in to `websocket_credential_rewrite` for client-to-server WebSocket text messages after an allowed `101` upgrade; server-to-client traffic and all other upgraded protocols remain raw passthrough. @@ -91,10 +91,24 @@ supervisor polls for config revisions and attempts to load new dynamic policy into the in-process OPA engine; CLI reads of the latest sandbox policy use the same effective configuration path. -If a new policy fails validation or loading, the supervisor reports the failure -and keeps the last-known-good policy. Static controls, such as filesystem -allowlists and process identity, require a new sandbox because they are applied -before the child process starts. +The supervisor validates complete effective policy generations before +activation. Overlapping endpoint selectors may contribute request allow and +deny rules only when their connection and request-processing metadata agree; +conflicting TLS, destination, credential, parser, or enforcement metadata +rejects the complete generation. + +The `[openshell.gateway] policy_validation_failure_mode` configuration controls +rejected generations. It defaults to `fail_closed`, which publishes a quarantine +generation, denies new egress, invalidates existing relays, and leaves the +previous policy inactive. Operators may explicitly select +`retain_last_valid`, which keeps the previous generation active. With no +previous valid generation, the effective mode remains `fail_closed` regardless +of the configured mode. The gateway distributes this startup configuration to +sandbox supervisors with each effective policy snapshot. OCSF configuration and finding events state the +candidate version, validation rationale, configured and effective modes, active +generation, and whether the previous policy is active. Static controls, +such as filesystem allowlists and process identity, require a new sandbox +because they are applied before the child process starts. Gateway-global policy can override sandbox-scoped policy. Use it sparingly because it changes the effective access model for every sandbox on the gateway. diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index c9f8a86b21..8d0df0fd6f 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -36,6 +36,43 @@ pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; +/// Gateway posture when a sandbox rejects a candidate policy generation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PolicyValidationFailureMode { + /// Deactivate the previous policy and deny new egress until a valid + /// generation is loaded. + #[default] + FailClosed, + /// Keep the last valid generation active when a newer candidate fails + /// validation. Startup still fails closed when no valid generation exists. + RetainLastValid, +} + +impl PolicyValidationFailureMode { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::FailClosed => "fail_closed", + Self::RetainLastValid => "retain_last_valid", + } + } +} + +impl FromStr for PolicyValidationFailureMode { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "fail_closed" => Ok(Self::FailClosed), + "retain_last_valid" => Ok(Self::RetainLastValid), + _ => Err(format!( + "invalid policy validation failure mode '{value}'; expected fail_closed or retain_last_valid" + )), + } + } +} + /// Default OCI repository for the supervisor image (no tag). pub const DEFAULT_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/supervisor"; @@ -386,6 +423,9 @@ pub struct Config { /// Log level (trace, debug, info, warn, error). pub log_level: String, + /// Security posture for rejected sandbox policy generations. + pub policy_validation_failure_mode: PolicyValidationFailureMode, + /// TLS configuration. When `None`, the server listens on plaintext HTTP. pub tls: Option, @@ -727,6 +767,7 @@ impl Config { health_bind_address: None, metrics_bind_address: None, log_level: default_log_level(), + policy_validation_failure_mode: PolicyValidationFailureMode::default(), tls, oidc: None, auth: GatewayAuthConfig::default(), @@ -971,9 +1012,10 @@ mod tests { use super::{ ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, detect_docker_socket_from_candidates, detect_driver, - docker_host_unix_socket_path, docker_socket_responds, is_unix_socket, - normalize_compute_driver_name, podman_socket_candidates_from_env, podman_socket_responds, + GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, + detect_docker_socket_from_candidates, detect_driver, docker_host_unix_socket_path, + docker_socket_responds, is_unix_socket, normalize_compute_driver_name, + podman_socket_candidates_from_env, podman_socket_responds, }; #[cfg(unix)] use std::io::{Read as _, Write as _}; @@ -1009,6 +1051,21 @@ mod tests { assert!(err.contains("unsupported compute driver 'firecracker'")); } + #[test] + fn policy_validation_failure_mode_is_secure_by_default() { + assert_eq!( + Config::new(None).policy_validation_failure_mode, + PolicyValidationFailureMode::FailClosed + ); + assert_eq!( + "retain_last_valid" + .parse::() + .unwrap(), + PolicyValidationFailureMode::RetainLastValid + ); + assert!("keep_old".parse::().is_err()); + } + #[test] fn compute_driver_name_normalization_accepts_builtin_and_custom_names() { assert_eq!(normalize_compute_driver_name(" VM ").unwrap(), "vm"); diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index b18cbb1293..3847e804b3 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -738,6 +738,8 @@ pub struct SettingsPollResult { pub global_policy_version: u32, pub provider_env_revision: u64, pub supervisor_middleware_services: Vec, + /// Gateway-configured posture for rejected policy generations. + pub policy_validation_failure_mode: crate::PolicyValidationFailureMode, } fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> SettingsPollResult { @@ -752,6 +754,41 @@ fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> Settin global_policy_version: inner.global_policy_version, provider_env_revision: inner.provider_env_revision, supervisor_middleware_services: inner.supervisor_middleware_services, + policy_validation_failure_mode: inner + .policy_validation_failure_mode + .parse() + .unwrap_or_default(), + } +} + +#[cfg(test)] +mod settings_poll_tests { + use super::settings_poll_result; + use crate::PolicyValidationFailureMode; + use crate::proto::GetSandboxConfigResponse; + + #[test] + fn validation_failure_mode_round_trips_from_gateway_config() { + let result = settings_poll_result(GetSandboxConfigResponse { + policy_validation_failure_mode: "retain_last_valid".to_string(), + ..Default::default() + }); + assert_eq!( + result.policy_validation_failure_mode, + PolicyValidationFailureMode::RetainLastValid + ); + } + + #[test] + fn unknown_validation_failure_mode_fails_closed() { + let result = settings_poll_result(GetSandboxConfigResponse { + policy_validation_failure_mode: "future_mode".to_string(), + ..Default::default() + }); + assert_eq!( + result.policy_validation_failure_mode, + PolicyValidationFailureMode::FailClosed + ); } } diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index a57acb0065..4bfea60747 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -44,7 +44,7 @@ pub use config::{ ComputeDriverKind, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, - MtlsAuthConfig, OidcConfig, TlsConfig, + MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, }; pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{GetResourceVersion, ObjectId, ObjectLabels, ObjectName, SetResourceVersion}; diff --git a/crates/openshell-policy/src/ambiguity.rs b/crates/openshell-policy/src/ambiguity.rs new file mode 100644 index 0000000000..16b4e3653a --- /dev/null +++ b/crates/openshell-policy/src/ambiguity.rs @@ -0,0 +1,590 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validation for endpoint selectors whose policy-derived behavior conflicts. + +use openshell_core::proto::{NetworkEndpoint, SandboxPolicy}; +use std::collections::{BTreeSet, HashSet, VecDeque}; +use std::fmt; + +/// One pair of endpoints that can authorize the same request but disagree on +/// policy-derived behavior that must have a single deterministic value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EndpointAmbiguity { + pub left_policy: String, + pub left_endpoint_index: usize, + pub left_selector: String, + pub right_policy: String, + pub right_endpoint_index: usize, + pub right_selector: String, + pub overlapping_ports: Vec, + pub conflicts: Vec, +} + +impl fmt::Display for EndpointAmbiguity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "network policies '{}' endpoint[{}] ({}) and '{}' endpoint[{}] ({}) overlap on port(s) {} with conflicting metadata: {}", + self.left_policy, + self.left_endpoint_index, + self.left_selector, + self.right_policy, + self.right_endpoint_index, + self.right_selector, + self.overlapping_ports + .iter() + .map(u32::to_string) + .collect::>() + .join(","), + self.conflicts.join("; "), + ) + } +} + +struct EndpointRef<'a> { + policy: &'a str, + index: usize, + endpoint: &'a NetworkEndpoint, +} + +/// Reject endpoint metadata ambiguity before a policy generation is activated. +/// +/// Request authorization rules (`access`, `rules`, and `deny_rules`) may be +/// contributed by multiple compatible endpoints. Metadata used to establish +/// or parse a connection must agree whenever the endpoint host, port, and (for +/// request-specific metadata) path selectors can match the same request. +#[must_use] +pub fn find_endpoint_ambiguities(policy: &SandboxPolicy) -> Vec { + let endpoints = policy + .network_policies + .iter() + .flat_map(|(key, rule)| { + let policy_name = if rule.name.is_empty() { + key.as_str() + } else { + rule.name.as_str() + }; + rule.endpoints + .iter() + .enumerate() + .map(move |(index, endpoint)| EndpointRef { + policy: policy_name, + index, + endpoint, + }) + }) + .collect::>(); + + let mut ambiguities = Vec::new(); + for left_index in 0..endpoints.len() { + for right_index in (left_index + 1)..endpoints.len() { + let left = &endpoints[left_index]; + let right = &endpoints[right_index]; + let overlapping_ports = overlapping_ports(left.endpoint, right.endpoint); + if overlapping_ports.is_empty() + || !host_patterns_overlap(&left.endpoint.host, &right.endpoint.host) + { + continue; + } + + let mut conflicts = connection_conflicts(left.endpoint, right.endpoint); + if path_patterns_overlap(&left.endpoint.path, &right.endpoint.path) { + conflicts.extend(request_pipeline_conflicts(left.endpoint, right.endpoint)); + } + if conflicts.is_empty() { + continue; + } + + ambiguities.push(EndpointAmbiguity { + left_policy: left.policy.to_string(), + left_endpoint_index: left.index, + left_selector: endpoint_selector(left.endpoint), + right_policy: right.policy.to_string(), + right_endpoint_index: right.index, + right_selector: endpoint_selector(right.endpoint), + overlapping_ports, + conflicts, + }); + } + } + ambiguities +} + +fn endpoint_selector(endpoint: &NetworkEndpoint) -> String { + let host = if endpoint.host.is_empty() { + "" + } else { + &endpoint.host + }; + let path = if endpoint.path.is_empty() { + "" + } else { + endpoint.path.as_str() + }; + format!("{host}:{}{}", display_ports(endpoint), path) +} + +fn display_ports(endpoint: &NetworkEndpoint) -> String { + effective_ports(endpoint) + .iter() + .map(u32::to_string) + .collect::>() + .join(",") +} + +fn effective_ports(endpoint: &NetworkEndpoint) -> BTreeSet { + if endpoint.ports.is_empty() { + (endpoint.port > 0) + .then_some(endpoint.port) + .into_iter() + .collect() + } else { + endpoint + .ports + .iter() + .copied() + .filter(|port| *port > 0) + .collect() + } +} + +fn overlapping_ports(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { + effective_ports(left) + .intersection(&effective_ports(right)) + .copied() + .collect() +} + +fn connection_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { + let mut conflicts = Vec::new(); + push_conflict( + &mut conflicts, + "tls", + &normalized_tls(&left.tls), + &normalized_tls(&right.tls), + ); + push_conflict( + &mut conflicts, + "allowed_ips", + &normalized_strings(&left.allowed_ips), + &normalized_strings(&right.allowed_ips), + ); + push_conflict( + &mut conflicts, + "advisor_proposed", + &left.advisor_proposed, + &right.advisor_proposed, + ); + conflicts +} + +fn request_pipeline_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { + let mut conflicts = Vec::new(); + push_conflict( + &mut conflicts, + "protocol", + &left.protocol.to_ascii_lowercase(), + &right.protocol.to_ascii_lowercase(), + ); + push_conflict( + &mut conflicts, + "enforcement", + &normalized_enforcement(&left.enforcement), + &normalized_enforcement(&right.enforcement), + ); + push_conflict( + &mut conflicts, + "allow_encoded_slash", + &left.allow_encoded_slash, + &right.allow_encoded_slash, + ); + push_conflict( + &mut conflicts, + "websocket_credential_rewrite", + &left.websocket_credential_rewrite, + &right.websocket_credential_rewrite, + ); + push_conflict( + &mut conflicts, + "request_body_credential_rewrite", + &left.request_body_credential_rewrite, + &right.request_body_credential_rewrite, + ); + push_conflict( + &mut conflicts, + "credential_signing", + &left.credential_signing, + &right.credential_signing, + ); + push_conflict( + &mut conflicts, + "signing_service", + &left.signing_service, + &right.signing_service, + ); + push_conflict( + &mut conflicts, + "signing_region", + &left.signing_region, + &right.signing_region, + ); + + if left.protocol.eq_ignore_ascii_case("graphql") + && right.protocol.eq_ignore_ascii_case("graphql") + { + push_conflict( + &mut conflicts, + "graphql_max_body_bytes", + &normalized_body_limit(left.graphql_max_body_bytes), + &normalized_body_limit(right.graphql_max_body_bytes), + ); + } + if is_json_rpc_family(&left.protocol) && is_json_rpc_family(&right.protocol) { + push_conflict( + &mut conflicts, + "json_rpc_max_body_bytes", + &normalized_body_limit(left.json_rpc_max_body_bytes), + &normalized_body_limit(right.json_rpc_max_body_bytes), + ); + } + if left.protocol.eq_ignore_ascii_case("mcp") && right.protocol.eq_ignore_ascii_case("mcp") { + push_conflict( + &mut conflicts, + "mcp.strict_tool_names", + &normalized_mcp_strict_tool_names(left), + &normalized_mcp_strict_tool_names(right), + ); + } + conflicts +} + +fn push_conflict( + conflicts: &mut Vec, + field: &str, + left: &T, + right: &T, +) { + if left != right { + conflicts.push(format!("{field}={left:?} vs {right:?}")); + } +} + +fn normalized_tls(value: &str) -> &'static str { + if value.eq_ignore_ascii_case("skip") { + "skip" + } else { + "auto" + } +} + +fn normalized_enforcement(value: &str) -> &'static str { + if value.eq_ignore_ascii_case("enforce") { + "enforce" + } else { + "audit" + } +} + +fn normalized_strings(values: &[String]) -> Vec { + values + .iter() + .map(|value| value.trim().to_ascii_lowercase()) + .collect::>() + .into_iter() + .collect() +} + +const DEFAULT_BODY_LIMIT: u32 = 65_536; + +fn normalized_body_limit(value: u32) -> u32 { + if value == 0 { + DEFAULT_BODY_LIMIT + } else { + value + } +} + +fn is_json_rpc_family(protocol: &str) -> bool { + protocol.eq_ignore_ascii_case("json-rpc") || protocol.eq_ignore_ascii_case("mcp") +} + +fn normalized_mcp_strict_tool_names(endpoint: &NetworkEndpoint) -> bool { + endpoint + .mcp + .as_ref() + .and_then(|options| options.strict_tool_names) + .unwrap_or(true) +} + +fn host_patterns_overlap(left: &str, right: &str) -> bool { + if left.is_empty() || right.is_empty() { + return true; + } + glob_patterns_overlap(&left.to_ascii_lowercase(), &right.to_ascii_lowercase(), '.') +} + +fn path_patterns_overlap(left: &str, right: &str) -> bool { + if left.is_empty() + || right.is_empty() + || matches!(left, "**" | "/**") + || matches!(right, "**" | "/**") + { + return true; + } + glob_patterns_overlap(left, right, '/') +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GlobToken { + Literal(char), + Star { crosses_delimiter: bool }, +} + +fn tokenize_glob(pattern: &str) -> Vec { + let chars = pattern.chars().collect::>(); + let mut tokens = Vec::new(); + let mut index = 0; + while index < chars.len() { + if chars[index] != '*' { + tokens.push(GlobToken::Literal(chars[index])); + index += 1; + continue; + } + + let start = index; + while index < chars.len() && chars[index] == '*' { + index += 1; + } + tokens.push(GlobToken::Star { + crosses_delimiter: index - start >= 2, + }); + } + tokens +} + +/// Decide whether two delimiter-aware glob languages intersect. +/// +/// This is a small product-NFA search. `*` consumes any character except the +/// delimiter and `**` consumes any character, including the delimiter. Star +/// epsilon transitions and self-loops make the state space finite. +fn glob_patterns_overlap(left: &str, right: &str, delimiter: char) -> bool { + let left = tokenize_glob(left); + let right = tokenize_glob(right); + let mut queue = VecDeque::from([(0_usize, 0_usize)]); + let mut seen = HashSet::new(); + + while let Some((left_index, right_index)) = queue.pop_front() { + if !seen.insert((left_index, right_index)) { + continue; + } + if left_index == left.len() && right_index == right.len() { + return true; + } + + if matches!(left.get(left_index), Some(GlobToken::Star { .. })) { + queue.push_back((left_index + 1, right_index)); + } + if matches!(right.get(right_index), Some(GlobToken::Star { .. })) { + queue.push_back((left_index, right_index + 1)); + } + + let Some(left_token) = left.get(left_index) else { + continue; + }; + let Some(right_token) = right.get(right_index) else { + continue; + }; + if tokens_share_character(*left_token, *right_token, delimiter) { + let next_left = if matches!(left_token, GlobToken::Star { .. }) { + left_index + } else { + left_index + 1 + }; + let next_right = if matches!(right_token, GlobToken::Star { .. }) { + right_index + } else { + right_index + 1 + }; + queue.push_back((next_left, next_right)); + } + } + false +} + +fn tokens_share_character(left: GlobToken, right: GlobToken, delimiter: char) -> bool { + match (left, right) { + (GlobToken::Literal(left), GlobToken::Literal(right)) => left == right, + (GlobToken::Literal(value), GlobToken::Star { crosses_delimiter }) + | (GlobToken::Star { crosses_delimiter }, GlobToken::Literal(value)) => { + crosses_delimiter || value != delimiter + } + ( + GlobToken::Star { + crosses_delimiter: _, + }, + GlobToken::Star { + crosses_delimiter: _, + }, + ) => true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::{NetworkBinary, NetworkPolicyRule}; + + fn endpoint(host: &str, port: u32) -> NetworkEndpoint { + NetworkEndpoint { + host: host.to_string(), + port, + ..Default::default() + } + } + + fn policy_with(left: NetworkEndpoint, right: NetworkEndpoint) -> SandboxPolicy { + let mut policy = SandboxPolicy::default(); + policy.network_policies.insert( + "left".to_string(), + NetworkPolicyRule { + name: "left".to_string(), + endpoints: vec![left], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + policy.network_policies.insert( + "right".to_string(), + NetworkPolicyRule { + name: "right".to_string(), + endpoints: vec![right], + binaries: vec![NetworkBinary { + path: "/usr/bin/bash".to_string(), + ..Default::default() + }], + }, + ); + policy + } + + #[test] + fn exact_and_wildcard_hosts_overlap() { + assert!(host_patterns_overlap("api.example.com", "*.example.com")); + assert!(host_patterns_overlap( + "us-aiplatform.googleapis.com", + "*-aiplatform.googleapis.com" + )); + assert!(!host_patterns_overlap("api.example.com", "*.other.com")); + } + + #[test] + fn intersecting_wildcards_are_detected() { + assert!(host_patterns_overlap("*.example.com", "api.*.com")); + assert!(host_patterns_overlap("**.example.com", "api.example.com")); + assert!(!host_patterns_overlap("*.example.com", "*.example.org")); + } + + #[test] + fn disjoint_ports_do_not_overlap() { + let mut left = endpoint("api.example.com", 443); + left.tls = "skip".to_string(); + let right = endpoint("api.example.com", 8443); + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn compatible_request_rules_may_overlap() { + let mut left = endpoint("api.example.com", 443); + left.protocol = "rest".to_string(); + left.tls = "skip".to_string(); + let mut right = left.clone(); + left.access = "read-only".to_string(); + right.access = "read-write".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn disjoint_path_specific_protocols_may_overlap() { + let mut left = endpoint("api.example.com", 443); + left.path = "/graphql".to_string(); + left.protocol = "graphql".to_string(); + let mut right = endpoint("api.example.com", 443); + right.path = "/repos/**".to_string(); + right.protocol = "rest".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn exact_wildcard_tls_conflict_is_rejected() { + let mut left = endpoint("*.example.com", 443); + left.tls = "skip".to_string(); + let right = endpoint("api.example.com", 443); + let ambiguities = find_endpoint_ambiguities(&policy_with(left, right)); + + assert_eq!(ambiguities.len(), 1); + assert!(ambiguities[0].conflicts[0].contains("tls")); + assert!(ambiguities[0].to_string().contains("left")); + assert!(ambiguities[0].to_string().contains("right")); + } + + #[test] + fn allowed_ip_conflict_is_rejected_regardless_of_order() { + let mut left = endpoint("api.example.com", 443); + left.allowed_ips = vec!["10.0.1.0/24".to_string(), "10.0.0.0/24".to_string()]; + let mut compatible = endpoint("api.example.com", 443); + compatible.allowed_ips = vec!["10.0.0.0/24".to_string(), "10.0.1.0/24".to_string()]; + assert!(find_endpoint_ambiguities(&policy_with(left.clone(), compatible)).is_empty()); + + let mut conflicting = endpoint("api.example.com", 443); + conflicting.allowed_ips = vec!["10.0.2.0/24".to_string()]; + let ambiguities = find_endpoint_ambiguities(&policy_with(left, conflicting)); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("allowed_ips")) + ); + } + + #[test] + fn credential_and_parser_conflicts_are_rejected_on_same_path() { + let mut left = endpoint("api.example.com", 443); + left.protocol = "rest".to_string(); + left.credential_signing = "sigv4".to_string(); + left.signing_service = "execute-api".to_string(); + let mut right = left.clone(); + right.signing_service = "bedrock".to_string(); + right.allow_encoded_slash = true; + + let ambiguities = find_endpoint_ambiguities(&policy_with(left, right)); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("signing_service")) + ); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("allow_encoded_slash")) + ); + } + + #[test] + fn different_binary_lists_do_not_hide_endpoint_ambiguity() { + let mut left = endpoint("api.example.com", 443); + left.tls = "skip".to_string(); + let right = endpoint("api.example.com", 443); + + assert_eq!( + find_endpoint_ambiguities(&policy_with(left, right)).len(), + 1 + ); + } +} diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 3c72b19b32..b6224c024f 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -17,6 +17,10 @@ use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::path::Path; +mod ambiguity; + +pub use ambiguity::{EndpointAmbiguity, find_endpoint_ambiguities}; + use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::proto::{ FilesystemPolicy, GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 78e5027953..bcf973930a 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -21,9 +21,12 @@ use std::sync::atomic::AtomicU32; use std::time::Duration; use tracing::{debug, info, warn}; +use openshell_core::PolicyValidationFailureMode; + use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, - DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, + DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, StateId, StatusId, + ocsf_emit, }; // --------------------------------------------------------------------------- @@ -834,7 +837,10 @@ fn load_policy_from_sidecar_bootstrap( policy, opa_engine, Some(proto), - LoadedPolicyOrigin::Gateway { revision: None }, + LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }, )) } @@ -1948,7 +1954,7 @@ async fn load_policy( } } - let loaded_policy_revision = + let mut loaded_policy_revision = policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); // Build OPA engine from baked-in rules + typed proto data. @@ -1958,12 +1964,37 @@ async fn load_policy( // container hasn't started yet. After the entrypoint spawns, the // engine is rebuilt with the real PID for symlink resolution. info!("Creating OPA engine from proto policy data"); + let mut has_last_valid_policy = true; let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => engine, + Ok(engine) => Arc::new(engine), Err(e) => { report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) .await; - return Err(e); + let validation_error = e.to_string(); + let candidate_version = snapshot.version; + let candidate_hash = snapshot.policy_hash.clone(); + // There is no in-memory last-known-good generation during + // startup, so both configured modes necessarily fail closed. + // Load the restrictive default atomically and keep the + // rejected revision unacknowledged for poll reconciliation. + has_last_valid_policy = false; + proto_policy = openshell_policy::restrictive_default_policy(); + let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); + let disposition = apply_policy_validation_failure( + &engine, + snapshot.policy_validation_failure_mode, + has_last_valid_policy, + candidate_version, + &validation_error, + )?; + emit_policy_validation_failure( + &disposition, + candidate_version, + &candidate_hash, + &validation_error, + ); + loaded_policy_revision = None; + engine } }; @@ -2006,7 +2037,7 @@ async fn load_policy( } else { MiddlewareRegistryStatus::Synchronized }; - let opa_engine = Some(Arc::new(engine)); + let opa_engine = Some(engine); let policy = match SandboxPolicy::try_from(proto_policy.clone()) { Ok(policy) => policy, @@ -2023,6 +2054,7 @@ async fn load_policy( middleware_registry_status, LoadedPolicyOrigin::Gateway { revision: loaded_policy_revision, + has_last_valid_policy, }, )); } @@ -2204,6 +2236,7 @@ enum LoadedPolicyOrigin { LocalOverride, Gateway { revision: Option, + has_last_valid_policy: bool, }, } @@ -2211,6 +2244,16 @@ impl LoadedPolicyOrigin { fn allows_gateway_policy_reload(&self) -> bool { matches!(self, Self::Gateway { .. }) } + + fn has_last_valid_policy(&self) -> bool { + match self { + Self::LocalOverride => true, + Self::Gateway { + has_last_valid_policy, + .. + } => *has_last_valid_policy, + } + } } impl LoadedPolicyRevision { @@ -2317,7 +2360,7 @@ fn initial_poll_disposition( ) -> InitialPollDisposition { match origin { LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision } => { + LoadedPolicyOrigin::Gateway { revision, .. } => { initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( InitialPollDisposition::Reconcile, InitialPollDisposition::Acknowledge, @@ -2547,6 +2590,150 @@ async fn reconcile_middleware_registry( } } +#[derive(Debug, PartialEq, Eq)] +struct PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode, + mode: PolicyValidationFailureMode, + previous_policy_active: bool, + active_generation: u64, +} + +struct RejectedPolicyGeneration { + version: u32, + policy_hash: String, + validation_error: String, + configured_mode: PolicyValidationFailureMode, +} + +fn apply_policy_validation_failure( + engine: &OpaEngine, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, + error: &str, +) -> Result { + let mode = if has_last_valid_policy { + configured_mode + } else { + PolicyValidationFailureMode::FailClosed + }; + match mode { + PolicyValidationFailureMode::FailClosed => { + let reason = format!( + "policy validation failed; fail-closed quarantine is active; candidate version {version} rejected: {error}" + ); + let active_generation = engine.enter_fail_closed(reason)?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: false, + active_generation, + }) + } + PolicyValidationFailureMode::RetainLastValid => { + let active_generation = engine.exit_fail_closed()?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: true, + active_generation, + }) + } + } +} + +fn policy_validation_failure_events( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) -> [OcsfEvent; 2] { + let previous_policy_state = if disposition.previous_policy_active { + "IS active" + } else { + "IS NOT active" + }; + let state = if disposition.previous_policy_active { + (StateId::Enabled, "retained_last_valid") + } else { + (StateId::Disabled, "fail_closed") + }; + let message = format!( + "Policy validation failed; configured_mode={} effective_mode={}; previous policy {previous_policy_state} [version:{version} active_generation:{} error:{error}]", + disposition.configured_mode.as_str(), + disposition.mode.as_str(), + disposition.active_generation, + ); + let finding_uid = format!("policy-validation-failed-{version}"); + let version_string = version.to_string(); + let config = ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(state.0, state.1) + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped( + "validation_failure_mode", + serde_json::json!(disposition.mode.as_str()), + ) + .unmapped( + "configured_validation_failure_mode", + serde_json::json!(disposition.configured_mode.as_str()), + ) + .unmapped( + "previous_policy_active", + serde_json::json!(disposition.previous_policy_active), + ) + .unmapped( + "active_generation", + serde_json::json!(disposition.active_generation), + ) + .unmapped("validation_error", serde_json::json!(error)) + .message(message.clone()) + .build(); + let finding = DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .is_alert(true) + .finding_info( + FindingInfo::new(&finding_uid, "Invalid policy generation rejected").with_desc(error), + ) + .evidence_pairs(&[ + ("candidate_version", &version_string), + ("candidate_policy_hash", policy_hash), + ("validation_failure_mode", disposition.mode.as_str()), + ( + "configured_validation_failure_mode", + disposition.configured_mode.as_str(), + ), + ( + "previous_policy_active", + if disposition.previous_policy_active { + "true" + } else { + "false" + }, + ), + ]) + .remediation("Submit a valid, unambiguous policy generation") + .message(message) + .build(); + [config, finding] +} + +fn emit_policy_validation_failure( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) { + for event in policy_validation_failure_events(disposition, version, policy_hash, error) { + ocsf_emit!(event); + } +} + async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::PolicySource; @@ -2571,6 +2758,8 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { > = std::collections::HashMap::new(); let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); let mut last_failed_runtime_revision: Option<(u64, String)> = None; + let mut rejected_policy_generation: Option = None; + let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); // A first poll that does not match the policy already loaded into OPA must // pass through the normal reconciliation path immediately. It must never @@ -2638,14 +2827,23 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { ¤t_middleware_services, &result.supervisor_middleware_services, ); - let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); + // A valid candidate may intentionally restore byte-for-byte policy + // content that was active before a rejected update. Its hash then + // equals `current_policy_hash`, but the runtime is still quarantined + // and must reload (or it would remain deny-all indefinitely). + let recovering_rejected_policy = reloads_gateway_policy + && rejected_policy_generation + .as_ref() + .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); + let policy_runtime_changed = recovering_rejected_policy + || gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); // A local policy override is not coupled to the gateway policy // snapshot, so its service registry can still be reconciled alone. @@ -2669,6 +2867,30 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { // Log which settings changed. log_setting_changes(¤t_settings, &result.settings); + // A posture change after a rejected update takes effect immediately. + // The compiled last-known-good engine remains available beneath a + // fail-closed quarantine, so an explicit retain_last_valid selection + // can reactivate it without accepting any part of the invalid policy. + if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { + let mode = result.policy_validation_failure_mode; + if mode != rejected.configured_mode { + let disposition = apply_policy_validation_failure( + &ctx.opa_engine, + mode, + has_last_valid_policy, + rejected.version, + &rejected.validation_error, + )?; + emit_policy_validation_failure( + &disposition, + rejected.version, + &rejected.policy_hash, + &rejected.validation_error, + ); + rejected.configured_mode = mode; + } + } + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) .status(StatusId::Success) @@ -2762,6 +2984,8 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .policy .as_ref() .expect("successful runtime reload requires a policy payload"); + has_last_valid_policy = true; + rejected_policy_generation = None; if policy_changed { if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { policy_local_ctx.set_current_policy(policy.clone()).await; @@ -2806,6 +3030,26 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { PolicyStatusUpdate::loaded(result.version), ); } + } else if recovering_rejected_policy + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully and fail-closed quarantine cleared [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); } if middleware_registry_changed { @@ -2832,6 +3076,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { Err(e) => { let failed_revision = (result.config_revision, result.policy_hash.clone()); if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + let validation_error = e.to_string(); ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Medium) .status(StatusId::Failure) @@ -2843,13 +3088,34 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { result.version )) .build()); + + let failure_mode = result.policy_validation_failure_mode; + let disposition = apply_policy_validation_failure( + &ctx.opa_engine, + failure_mode, + has_last_valid_policy, + result.version, + &validation_error, + )?; + emit_policy_validation_failure( + &disposition, + result.version, + &result.policy_hash, + &validation_error, + ); + rejected_policy_generation = Some(RejectedPolicyGeneration { + version: result.version, + policy_hash: result.policy_hash.clone(), + validation_error: validation_error.clone(), + configured_mode: failure_mode, + }); if policy_changed && result.version > 0 && result.policy_source == PolicySource::Sandbox { enqueue_policy_status( &status_sender, - PolicyStatusUpdate::failed(result.version, e.to_string()), + PolicyStatusUpdate::failed(result.version, validation_error), ); } } @@ -3273,6 +3539,7 @@ filesystem_policy: global_policy_version: 0, provider_env_revision: 0, supervisor_middleware_services: Vec::new(), + policy_validation_failure_mode: PolicyValidationFailureMode::default(), } } @@ -3499,6 +3766,7 @@ filesystem_policy: initial_poll_disposition( &LoadedPolicyOrigin::Gateway { revision: Some(loaded), + has_last_valid_policy: true, }, &canonical, ), @@ -3528,7 +3796,10 @@ filesystem_policy: 2, openshell_core::proto::PolicySource::Sandbox, ); - let origin = LoadedPolicyOrigin::Gateway { revision: None }; + let origin = LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }; assert_eq!( initial_poll_disposition(&origin, &canonical), @@ -3551,4 +3822,168 @@ filesystem_policy: ); } } + + #[test] + fn fail_closed_validation_failure_deactivates_previous_generation() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(!disposition.previous_policy_active); + assert!(disposition.active_generation > previous_generation); + assert!( + engine + .fail_closed_reason() + .expect("quarantine reason") + .contains("candidate version 7 rejected") + ); + } + + #[test] + fn retain_validation_failure_keeps_previous_generation_active() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let quarantined = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 6, + "conflicting tls metadata", + ) + .unwrap(); + assert!(!quarantined.previous_policy_active); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(disposition.previous_policy_active); + assert!(disposition.active_generation > quarantined.active_generation); + assert!(disposition.active_generation > previous_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[test] + fn retain_validation_failure_without_last_valid_policy_stays_fail_closed() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + false, + 1, + "conflicting tls metadata", + ) + .unwrap(); + + assert_eq!( + disposition.configured_mode, + PolicyValidationFailureMode::RetainLastValid + ); + assert_eq!(disposition.mode, PolicyValidationFailureMode::FailClosed); + assert!(!disposition.previous_policy_active); + assert!(engine.fail_closed_reason().is_some()); + + let [config, _] = policy_validation_failure_events( + &disposition, + 1, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "retain_last_valid" + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + } + + #[test] + fn validation_failure_ocsf_states_whether_previous_policy_is_active() { + let fail_closed = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::FailClosed, + mode: PolicyValidationFailureMode::FailClosed, + previous_policy_active: false, + active_generation: 9, + }; + let [config, finding] = policy_validation_failure_events( + &fail_closed, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["class_uid"], 5019); + assert_eq!(config["status"], "Failure"); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "fail_closed" + ); + assert_eq!(config["unmapped"]["previous_policy_active"], false); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + + let finding = finding.to_json().unwrap(); + assert_eq!(finding["class_uid"], 2004); + assert_eq!(finding["action"], "Denied"); + assert_eq!(finding["disposition"], "Blocked"); + + let retained = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::RetainLastValid, + mode: PolicyValidationFailureMode::RetainLastValid, + previous_policy_active: true, + active_generation: 4, + }; + let [config, _] = policy_validation_failure_events( + &retained, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["previous_policy_active"], true); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS active") + ); + } } diff --git a/crates/openshell-sandbox/src/metadata_server.rs b/crates/openshell-sandbox/src/metadata_server.rs index aff1b5418a..cba614e496 100644 --- a/crates/openshell-sandbox/src/metadata_server.rs +++ b/crates/openshell-sandbox/src/metadata_server.rs @@ -135,3 +135,93 @@ async fn handle_connection( Ok(()) }) } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::sync::mpsc; + + struct RecordingHandler { + requests: mpsc::UnboundedSender<(String, String)>, + } + + impl MetadataHandler for RecordingHandler { + async fn handle( + &self, + method: &str, + path: &str, + _request: &[u8], + stream: &mut S, + ) -> Result<()> { + self.requests + .send((method.to_string(), path.to_string())) + .unwrap(); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .map_err(|error| miette::miette!("{error}"))?; + Ok(()) + } + } + + async fn connection_pair() -> (tokio::net::TcpStream, tokio::net::TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = tokio::net::TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + (client, server) + } + + #[tokio::test] + async fn metadata_loopback_dispatches_method_path_and_response() { + let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); + let handler = RecordingHandler { + requests: requests_tx, + }; + let (mut client, server) = connection_pair().await; + let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); + + client + .write_all(b"GET /computeMetadata/v1/instance HTTP/1.1\r\nHost: metadata\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + server_task.await.unwrap().unwrap(); + + assert_eq!( + requests_rx.try_recv().unwrap(), + ( + "GET".to_string(), + "/computeMetadata/v1/instance".to_string() + ) + ); + assert_eq!(response, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); + } + + #[tokio::test] + async fn metadata_loopback_rejects_oversized_headers_before_handler() { + let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); + let handler = RecordingHandler { + requests: requests_tx, + }; + let (mut client, server) = connection_pair().await; + let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); + + client + .write_all(&vec![b'x'; MAX_REQUEST_BYTES]) + .await + .unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + server_task.await.unwrap().unwrap(); + + assert_eq!( + response, + b"HTTP/1.1 413 Request Entity Too Large\r\nContent-Length: 0\r\n\r\n" + ); + assert!(requests_rx.try_recv().is_err()); + } +} diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 269b9f40e9..0a66a7bd92 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -404,6 +404,13 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result, #[serde(default)] pub grpc_rate_limit_window_seconds: Option, + /// Security posture when a sandbox rejects a candidate policy generation. + #[serde(default)] + pub policy_validation_failure_mode: Option, // ── Service routing ────────────────────────────────────────────────── /// Subject Alternative Names configured on the gateway server certificate. @@ -403,6 +406,7 @@ compute_drivers = ["kubernetes"] sandbox_namespace = "agents" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 +policy_validation_failure_mode = "retain_last_valid" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" client_tls_secret_name = "openshell-sandbox-tls" @@ -431,11 +435,27 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ); assert_eq!(gw.grpc_rate_limit_requests, Some(120)); assert_eq!(gw.grpc_rate_limit_window_seconds, Some(60)); + assert_eq!( + gw.policy_validation_failure_mode, + Some(openshell_core::PolicyValidationFailureMode::RetainLastValid) + ); assert!(gw.tls.is_some()); assert!(gw.oidc.is_some()); assert!(file.openshell.drivers.contains_key("kubernetes")); } + #[test] + fn rejects_unknown_policy_validation_failure_mode() { + let tmp = write_tmp( + r#" +[openshell.gateway] +policy_validation_failure_mode = "keep_old" +"#, + ); + let error = load(tmp.path()).expect_err("unknown posture must fail TOML validation"); + assert!(error.to_string().contains("policy_validation_failure_mode")); + } + #[test] fn parses_gateway_auth_config() { let toml = r" diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 59774bbcc8..fba2362181 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1362,11 +1362,12 @@ pub(super) async fn handle_get_sandbox_config( let settings = merge_effective_settings(&global_settings, &sandbox_settings)?; let supervisor_middleware_services = state.middleware_registry.required_services(policy.as_ref()); - let config_revision = compute_config_revision( + let config_revision = compute_config_revision_with_validation_mode( policy.as_ref(), &settings, policy_source, &supervisor_middleware_services, + state.config.policy_validation_failure_mode, ); let provider_env_revision = compute_provider_env_revision_with_catalog( state.store.as_ref(), @@ -1385,6 +1386,11 @@ pub(super) async fn handle_get_sandbox_config( global_policy_version, provider_env_revision, supervisor_middleware_services, + policy_validation_failure_mode: state + .config + .policy_validation_failure_mode + .as_str() + .to_string(), })) } @@ -1639,7 +1645,6 @@ async fn handle_update_config_inner( "one of policy, setting_key, or merge_operations must be provided", )); } - if req.global { if !req.annotations.is_empty() { return Err(Status::invalid_argument( @@ -3391,14 +3396,16 @@ fn deterministic_policy_hash(policy: &ProtoSandboxPolicy) -> String { } /// Compute a fingerprint for the effective sandbox configuration. -fn compute_config_revision( +fn compute_config_revision_with_validation_mode( policy: Option<&ProtoSandboxPolicy>, settings: &HashMap, policy_source: PolicySource, supervisor_middleware_services: &[openshell_core::proto::SupervisorMiddlewareService], + policy_validation_failure_mode: openshell_core::PolicyValidationFailureMode, ) -> u64 { let mut hasher = Sha256::new(); hasher.update((policy_source as i32).to_le_bytes()); + hasher.update(policy_validation_failure_mode.as_str().as_bytes()); if let Some(policy) = policy { hasher.update(deterministic_policy_hash(policy).as_bytes()); } @@ -3440,6 +3447,22 @@ fn compute_config_revision( u64::from_le_bytes(bytes) } +#[cfg(test)] +fn compute_config_revision( + policy: Option<&ProtoSandboxPolicy>, + settings: &HashMap, + policy_source: PolicySource, + supervisor_middleware_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> u64 { + compute_config_revision_with_validation_mode( + policy, + settings, + policy_source, + supervisor_middleware_services, + openshell_core::PolicyValidationFailureMode::default(), + ) +} + fn draft_chunk_record_to_proto(record: &DraftChunkRecord) -> Result { use openshell_core::proto::NetworkPolicyRule; @@ -10300,6 +10323,28 @@ mod tests { assert_ne!(rev_a, rev_b); } + #[test] + fn config_revision_changes_when_validation_failure_mode_changes() { + let policy = ProtoSandboxPolicy::default(); + let settings = HashMap::new(); + + let fail_closed = compute_config_revision_with_validation_mode( + Some(&policy), + &settings, + PolicySource::Sandbox, + &[], + openshell_core::PolicyValidationFailureMode::FailClosed, + ); + let retain_last_valid = compute_config_revision_with_validation_mode( + Some(&policy), + &settings, + PolicySource::Sandbox, + &[], + openshell_core::PolicyValidationFailureMode::RetainLastValid, + ); + assert_ne!(fail_closed, retain_last_valid); + } + #[test] fn config_revision_changes_when_supervisor_middleware_services_change() { let policy = ProtoSandboxPolicy::default(); diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index c6c1af5aed..6ea71a3e8a 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -19,3 +19,63 @@ pub mod run; pub mod sigv4; mod spiffe_endpoint; mod token_grant; + +#[cfg(test)] +pub(crate) mod test_alloc { + use std::alloc::{GlobalAlloc, Layout, System}; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct CountingAllocator; + + static ALLOCATIONS: AtomicU64 = AtomicU64::new(0); + static ALLOCATED_BYTES: AtomicU64 = AtomicU64::new(0); + + #[allow(unsafe_code)] + unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let pointer = unsafe { System.alloc(layout) }; + if !pointer.is_null() { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(layout.size() as u64, Ordering::Relaxed); + } + pointer + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let pointer = unsafe { System.alloc_zeroed(layout) }; + if !pointer.is_null() { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(layout.size() as u64, Ordering::Relaxed); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + unsafe { System.dealloc(pointer, layout) }; + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let pointer = unsafe { System.realloc(pointer, layout, new_size) }; + if !pointer.is_null() { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(new_size as u64, Ordering::Relaxed); + } + pointer + } + } + + #[global_allocator] + static GLOBAL: CountingAllocator = CountingAllocator; + + pub fn reset() { + ALLOCATIONS.store(0, Ordering::SeqCst); + ALLOCATED_BYTES.store(0, Ordering::SeqCst); + } + + pub fn snapshot() -> (u64, u64) { + ( + ALLOCATIONS.load(Ordering::SeqCst), + ALLOCATED_BYTES.load(Ordering::SeqCst), + ) + } +} diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 313548aedf..67a6070136 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -20,6 +20,7 @@ use std::sync::{ Arc, Mutex, RwLock, atomic::{AtomicU64, Ordering}, }; +use tokio::sync::watch; use tracing::info; /// Baked-in rego rules for OPA policy evaluation. @@ -123,6 +124,26 @@ pub struct OpaEngine { engine: Mutex, generation: Arc, middleware_runner: RwLock, + generation_tx: watch::Sender, + fail_closed_reason: RwLock>, +} + +#[cfg(test)] +static TEST_OPA_QUERY_COUNT: AtomicU64 = AtomicU64::new(0); + +#[cfg(test)] +fn record_test_opa_query() { + TEST_OPA_QUERY_COUNT.fetch_add(1, Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn reset_test_opa_query_count() { + TEST_OPA_QUERY_COUNT.store(0, Ordering::SeqCst); +} + +#[cfg(test)] +pub(crate) fn test_opa_query_count() -> u64 { + TEST_OPA_QUERY_COUNT.load(Ordering::SeqCst) } /// Generation guard captured when an HTTP tunnel or request path starts. @@ -130,6 +151,7 @@ pub struct OpaEngine { pub struct PolicyGenerationGuard { captured_generation: u64, current_generation: Arc, + generation_rx: watch::Receiver, } impl PolicyGenerationGuard { @@ -155,6 +177,19 @@ impl PolicyGenerationGuard { } Ok(()) } + + /// Wait until the policy generation changes. + /// + /// Relay boundaries use this to close even an idle or raw stream as soon + /// as a new generation (including fail-closed quarantine) is published. + pub async fn wait_until_stale(&self) { + let mut receiver = self.generation_rx.clone(); + while !self.is_stale() { + if receiver.changed().await.is_err() { + return; + } + } + } } /// Per-tunnel L7 policy evaluator bound to the engine generation captured when @@ -201,6 +236,33 @@ impl TunnelPolicyEngine { } impl OpaEngine { + fn with_engine(engine: regorus::Engine) -> Self { + let generation = Arc::new(AtomicU64::new(0)); + let (generation_tx, _) = watch::channel(0); + Self { + engine: Mutex::new(engine), + generation, + middleware_runner: RwLock::new(ChainRunner::default()), + generation_tx, + fail_closed_reason: RwLock::new(None), + } + } + + fn advance_generation(&self) -> u64 { + let generation = self.generation.fetch_add(1, Ordering::AcqRel) + 1; + self.generation_tx.send_replace(generation); + generation + } + + #[cfg(test)] + pub(crate) fn poison_lock_for_test(&self) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = self.engine.lock().expect("test engine lock"); + panic!("poison OPA engine lock for compatibility fallback test"); + })); + assert!(self.engine.is_poisoned()); + } + /// Load policy from a `.rego` rules file and data from a YAML file. /// /// Preprocesses the YAML data to expand access presets and validate L7 config. @@ -232,11 +294,7 @@ impl OpaEngine { engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; - Ok(Self { - engine: Mutex::new(engine), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }) + Ok(Self::with_engine(engine)) } /// Load policy rules and data from strings (data is YAML). @@ -287,11 +345,7 @@ impl OpaEngine { engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; - Ok(Self { - engine: Mutex::new(engine), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }) + Ok(Self::with_engine(engine)) } /// Create OPA engine from a typed proto policy. @@ -325,6 +379,18 @@ impl OpaEngine { entrypoint_pid: u32, require_binary_identity: bool, ) -> Result { + let ambiguities = openshell_policy::find_endpoint_ambiguities(proto); + if !ambiguities.is_empty() { + return Err(miette::miette!( + "network endpoint ambiguity validation failed:\n{}", + ambiguities + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + )); + } + emit_binary_identity_mode(require_binary_identity, "proto"); if let Err(violations) = openshell_policy::validate_sandbox_policy(proto) { let errors = violations @@ -375,11 +441,7 @@ impl OpaEngine { engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; - Ok(Self { - engine: Mutex::new(engine), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }) + Ok(Self::with_engine(engine)) } /// Evaluate a network access request against the loaded policy. @@ -395,6 +457,19 @@ impl OpaEngine { .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + let fail_closed_reason = self + .fail_closed_reason + .read() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? + .clone(); + if let Some(reason) = fail_closed_reason { + return Ok(PolicyDecision { + allowed: false, + reason, + matched_policy: None, + }); + } + engine .set_input_json(&input_json.to_string()) .map_err(|e| miette::miette!("{e}"))?; @@ -438,6 +513,9 @@ impl OpaEngine { &self, input: &NetworkInput, ) -> Result<(NetworkAction, u64)> { + #[cfg(test)] + record_test_opa_query(); + let input_json = network_input_json(input); let mut engine = self @@ -446,6 +524,15 @@ impl OpaEngine { .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; let generation = self.current_generation(); + let fail_closed_reason = self + .fail_closed_reason + .read() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? + .clone(); + if let Some(reason) = fail_closed_reason { + return Ok((NetworkAction::Deny { reason }, generation)); + } + engine .set_input_json(&input_json.to_string()) .map_err(|e| miette::miette!("{e}"))?; @@ -492,7 +579,11 @@ impl OpaEngine { .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; *engine = new_engine; - self.generation.fetch_add(1, Ordering::AcqRel); + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = None; + self.advance_generation(); Ok(()) } @@ -527,7 +618,11 @@ impl OpaEngine { .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; *engine = new_engine; - self.generation.fetch_add(1, Ordering::AcqRel); + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = None; + self.advance_generation(); Ok(()) } @@ -562,10 +657,61 @@ impl OpaEngine { .map_err(|_| miette::miette!("middleware runner lock poisoned"))?; *engine = new_engine; *runner = new_runner; - self.generation.fetch_add(1, Ordering::AcqRel); + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = None; + self.advance_generation(); Ok(()) } + /// Publish a deny-all quarantine generation without activating any part + /// of the invalid candidate policy. + /// + /// The existing compiled engine remains available for an explicit + /// `retain_last_valid` posture or a later valid reload, but all new network + /// decisions deny with `reason` while the quarantine is active. Advancing + /// the generation invalidates and wakes every pinned relay. + pub fn enter_fail_closed(&self, reason: impl Into) -> Result { + let _engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = + Some(reason.into()); + Ok(self.advance_generation()) + } + + pub fn fail_closed_reason(&self) -> Option { + self.fail_closed_reason + .read() + .ok() + .and_then(|reason| reason.clone()) + } + + /// Reactivate the compiled last-known-good engine after an operator + /// explicitly selects the availability-oriented retention posture. + pub fn exit_fail_closed(&self) -> Result { + let _engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + let was_fail_closed = self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? + .take() + .is_some(); + if was_fail_closed { + Ok(self.advance_generation()) + } else { + Ok(self.current_generation()) + } + } + /// Current policy generation. Successful reloads increment this value. pub fn current_generation(&self) -> u64 { self.generation.load(Ordering::Acquire) @@ -579,7 +725,7 @@ impl OpaEngine { .write() .map_err(|_| miette::miette!("middleware runner lock poisoned"))?; *runner = ChainRunner::from_registry(registry); - self.generation.fetch_add(1, Ordering::AcqRel); + self.advance_generation(); Ok(()) } @@ -612,6 +758,7 @@ impl OpaEngine { Ok(PolicyGenerationGuard { captured_generation: generation, current_generation: Arc::clone(&self.generation), + generation_rx: self.generation_tx.subscribe(), }) } @@ -674,6 +821,9 @@ impl OpaEngine { &self, input: &NetworkInput, ) -> Result<(Vec, u64)> { + #[cfg(test)] + record_test_opa_query(); + let input_json = network_input_json(input); let mut engine = self @@ -732,6 +882,9 @@ impl OpaEngine { /// denial while preserving separate handling for `allowed_ips` and advisor /// proposals. pub fn query_exact_declared_endpoint_host(&self, input: &NetworkInput) -> Result { + #[cfg(test)] + record_test_opa_query(); + let input_json = network_input_json(input); let mut engine = self @@ -771,6 +924,7 @@ impl OpaEngine { generation_guard: PolicyGenerationGuard { captured_generation: generation, current_generation: Arc::clone(&self.generation), + generation_rx: self.generation_tx.subscribe(), }, middleware_runner: self.middleware_runner()?, }) @@ -3047,11 +3201,7 @@ network_policies: .expect("policy should load"); rego.add_data_json(&data_json.to_string()) .expect("data should load"); - let engine = OpaEngine { - engine: Mutex::new(rego), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }; + let engine = OpaEngine::with_engine(rego); let input = l7_websocket_graphql_input( "realtime.graphql.com", serde_json::json!([{ @@ -4625,6 +4775,97 @@ network_policies: assert_eq!(val, regorus::Value::from(true)); } + #[test] + fn proto_load_rejects_ambiguous_endpoint_metadata_with_rationale() { + let mut policy = ProtoSandboxPolicy::default(); + policy.network_policies.insert( + "wildcard".to_string(), + NetworkPolicyRule { + name: "wildcard".to_string(), + endpoints: vec![NetworkEndpoint { + host: "*.example.com".to_string(), + port: 443, + tls: "skip".to_string(), + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + policy.network_policies.insert( + "exact".to_string(), + NetworkPolicyRule { + name: "exact".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/bash".to_string(), + ..Default::default() + }], + }, + ); + + let Err(error) = OpaEngine::from_proto(&policy) else { + panic!("ambiguity must reject activation"); + }; + let message = error.to_string(); + assert!(message.contains("ambiguity validation failed")); + assert!(message.contains("wildcard")); + assert!(message.contains("exact")); + assert!(message.contains("tls")); + } + + #[tokio::test] + async fn fail_closed_quarantine_denies_and_wakes_generation_guards() { + let engine = test_engine(); + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let stale = guard.wait_until_stale(); + + let generation = engine + .enter_fail_closed("candidate policy validation failed: conflicting tls") + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), stale) + .await + .expect("generation waiter should wake"); + assert_eq!(generation, 1); + assert!(guard.is_stale()); + + let input = NetworkInput { + host: "api.anthropic.com".to_string(), + port: 443, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let action = engine.evaluate_network_action(&input).unwrap(); + assert_eq!( + action, + NetworkAction::Deny { + reason: "candidate policy validation failed: conflicting tls".to_string() + } + ); + } + + #[test] + fn valid_reload_exits_fail_closed_quarantine() { + let engine = test_engine(); + engine.enter_fail_closed("invalid candidate").unwrap(); + assert!(engine.fail_closed_reason().is_some()); + + engine.reload(TEST_POLICY, TEST_DATA_YAML).unwrap(); + + assert!(engine.fail_closed_reason().is_none()); + assert_eq!(engine.current_generation(), 2); + } + #[test] fn endpoint_config_generation_matches_query_generation() { let engine = l7_engine(); @@ -4952,6 +5193,7 @@ network_policies: port: 8567 protocol: rest enforcement: enforce + tls: skip allowed_ips: - 192.168.1.100 rules: @@ -4990,7 +5232,7 @@ process: } #[test] - fn overlapping_policies_endpoint_config_returns_result() { + fn overlapping_policy_outputs_are_snapshotted_independently() { let engine = OpaEngine::from_strings(TEST_POLICY, OVERLAPPING_L7_TEST_DATA) .expect("engine should load overlapping data"); let input = NetworkInput { @@ -5001,12 +5243,29 @@ process: ancestors: vec![], cmdline_paths: vec![], }; - // Should return config from one of the entries without error. - let config = engine.query_endpoint_config(&input).unwrap(); - assert!( - config.is_some(), - "Expected endpoint config for overlapping policies" + assert_eq!( + engine.evaluate_network_action(&input).unwrap(), + NetworkAction::Allow { + matched_policy: Some("allow_192_168_1_100_8567".to_string()) + } ); + + let (configs, generation) = engine + .query_endpoint_configs_with_generation(&input) + .unwrap(); + assert_eq!(generation, engine.current_generation()); + assert_eq!(configs.len(), 2); + assert_eq!(get_str(&configs[0], "tls").as_deref(), Some("skip")); + assert_eq!(get_str_array(&configs[0], "allowed_ips"), ["192.168.1.100"]); + assert_eq!(get_str(&configs[1], "tls"), None); + + let selected = engine.query_endpoint_config(&input).unwrap().unwrap(); + assert_eq!( + crate::l7::parse_tls_mode(&selected), + crate::l7::TlsMode::Skip + ); + assert_eq!(engine.query_allowed_ips(&input).unwrap(), ["192.168.1.100"]); + assert!(engine.query_exact_declared_endpoint_host(&input).unwrap()); } // ======================================================================== diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index ab2313b890..9588baa388 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3,6 +3,10 @@ //! HTTP CONNECT proxy with OPA policy evaluation and process-identity binding. +mod destination; +mod egress; +mod relay; + use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; @@ -30,6 +34,15 @@ use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; +use self::destination::{ + DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, + validate_destination, +}; +use self::egress::{ + EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, + L7RouteSnapshot, ProcessIdentityEvidence, +}; + const MAX_HEADER_BYTES: usize = 8192; const TUNNEL_PROTOCOL_PEEK_BYTES: usize = crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE.len(); #[cfg(not(test))] @@ -81,21 +94,6 @@ const CHUNK_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1 #[cfg(test)] const CHUNK_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(100); -/// Result of a proxy CONNECT policy decision. -struct ConnectDecision { - action: NetworkAction, - /// Policy generation used for the L4 network decision. - generation: u64, - /// Resolved binary path. - binary: Option, - /// PID owning the socket. - binary_pid: Option, - /// Ancestor binary paths from process tree walk. - ancestors: Vec, - /// Cmdline-derived absolute paths (for script detection). - cmdline_paths: Vec, -} - /// Outcome of an inference interception attempt. /// /// Returned by [`handle_inference_interception`] so the call site can emit @@ -563,7 +561,7 @@ fn emit_denial( host: &str, port: u16, binary: &str, - decision: &ConnectDecision, + decision: &EgressDecision, reason: &str, stage: &str, ) { @@ -592,7 +590,7 @@ fn emit_denial_simple( host: &str, port: u16, binary: &str, - decision: &ConnectDecision, + decision: &EgressDecision, reason: &str, stage: &str, ) { @@ -614,6 +612,245 @@ fn emit_denial_simple( } } +#[allow(clippy::too_many_arguments)] +fn build_connect_allow_ocsf_event( + peer_addr: SocketAddr, + host: &str, + port: u16, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, + l7_inspection: bool, +) -> openshell_ocsf::OcsfEvent { + let connect_msg = if l7_inspection { + "CONNECT_L7" + } else { + "CONNECT" + }; + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "opa") + .message(format!("{connect_msg} allowed {host}:{port}")) + .build() +} + +#[allow(clippy::too_many_arguments)] +fn build_forward_allow_ocsf_event( + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, +) -> openshell_ocsf::OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "opa") + .message(format!("FORWARD allowed {method} {host}:{port}{path}")) + .build() +} + +#[allow(clippy::too_many_arguments)] +fn build_forward_policy_deny_ocsf_event( + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + reason: &str, +) -> openshell_ocsf::OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule("-", "opa") + .message(format!("FORWARD denied {method} {host}:{port}{path}")) + .status_detail(reason) + .build() +} + +#[allow(clippy::too_many_arguments)] +async fn deny_connect_destination( + client: &mut TcpStream, + denial: &DestinationDenial, + peer_addr: SocketAddr, + host: &str, + port: u16, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + decision: &EgressDecision, + denial_tx: &Option>, + activity_tx: &Option, +) -> Result<()> { + let detail = match denial.kind { + DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", + DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", + DestinationDenialKind::AllowedIps => "allowed_ips check failed", + DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", + DestinationDenialKind::InternalAddress => "internal address", + }; + let message = if denial.kind == DestinationDenialKind::InternalAddress { + format!("CONNECT blocked: internal address {host}:{port}") + } else { + format!("CONNECT blocked: {detail} for {host}:{port}") + }; + + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule("-", "ssrf") + .message(message) + .status_detail(&denial.reason) + .build(); + ocsf_emit!(event); + + emit_denial( + denial_tx, + host, + port, + binary, + decision, + &denial.reason, + "ssrf", + ); + // Preserve the current activity contract. The declared-endpoint branch + // historically emits the denial without a separate SSRF activity count. + if denial.kind != DestinationDenialKind::DeclaredEndpoint { + emit_activity(activity_tx, true, "ssrf"); + } + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!("CONNECT {host}:{port} blocked: {detail}"), + ), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn deny_forward_destination( + client: &mut TcpStream, + denial: &DestinationDenial, + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, + decision: &EgressDecision, + denial_tx: Option<&mpsc::UnboundedSender>, + activity_tx: Option<&ActivitySender>, +) -> Result<()> { + let detail = match denial.kind { + DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", + DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", + DestinationDenialKind::AllowedIps => "allowed_ips check failed", + DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", + DestinationDenialKind::InternalAddress => "internal address", + }; + let log_detail = if denial.kind == DestinationDenialKind::InternalAddress { + "internal IP without allowed_ips" + } else { + detail + }; + + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "ssrf") + .message(format!("FORWARD blocked: {log_detail} for {host}:{port}")) + .status_detail(&denial.reason) + .build(); + ocsf_emit!(event); + + emit_denial_simple( + denial_tx, + host, + port, + binary, + decision, + &denial.reason, + "ssrf", + ); + // Preserve the current activity contract. The declared-endpoint branch + // historically emits the denial without a separate SSRF activity count. + if denial.kind != DestinationDenialKind::DeclaredEndpoint { + emit_activity_simple(activity_tx, true, "ssrf"); + } + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!("{method} {host}:{port} blocked: {detail}"), + ), + ) + .await +} + // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. @@ -742,20 +979,19 @@ async fn handle_tcp_connection( let opa_clone = opa_engine.clone(); let cache_clone = identity_cache.clone(); let pid_clone = entrypoint_pid.clone(); - let host_clone = host_lc.clone(); - let decision = tokio::task::spawn_blocking(move || { - evaluate_opa_tcp( - peer_addr, - &opa_clone, - &cache_clone, - &pid_clone, - &host_clone, - port, - ) + let intent = EgressIntent::connect(host_lc.clone(), port); + let mut decision = tokio::task::spawn_blocking(move || { + authorize_egress_intent(peer_addr, &opa_clone, &cache_clone, &pid_clone, intent) }) .await .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + debug!( + transport = ?decision.intent.transport, + identity = ?decision.identity, + "Authorized explicit proxy egress intent" + ); + // Extract action string and matched policy for logging let (matched_policy, deny_reason) = match &decision.action { NetworkAction::Allow { matched_policy } => (matched_policy.clone(), String::new()), @@ -842,288 +1078,66 @@ async fn handle_tcp_connection( // allowed_ips validation below — so an internal-address CONNECT still gets // the SSRF 403 and telemetry in degraded state — but before the upstream // connect and before `200 Connection Established`. - let effective_tls_skip = - query_tls_mode(&opa_engine, &decision, &host_lc, port) == crate::l7::TlsMode::Skip; + hydrate_tls_mode(&opa_engine, &mut decision); + let effective_tls_skip = decision.endpoint.tls_mode == crate::l7::TlsMode::Skip; let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - // Query allowed_ips from the matched endpoint config (if any). - // When present, the SSRF check validates resolved IPs against this - // allowlist instead of blanket-blocking all private IPs. - // When the policy host is already a literal IP address, treat it as - // implicitly allowed — the user explicitly declared the destination. - // Exact declared hostnames also skip the private-IP blanket block below, - // while keeping loopback/link-local/unspecified addresses denied. - let mut raw_allowed_ips = query_allowed_ips(&opa_engine, &decision, &host_lc, port); - if raw_allowed_ips.is_empty() { - raw_allowed_ips = implicit_allowed_ips_for_ip_host(&host); + match hydrate_destination_plan(&opa_engine, &mut decision, *trusted_host_gateway) { + Ok(()) => {} + Err(denial) => { + deny_connect_destination( + &mut client, + &denial, + peer_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + &decision, + &denial_tx, + &activity_tx, + ) + .await?; + return Ok(()); + } } - let exact_declared_endpoint_host = - query_exact_declared_endpoint_host(&opa_engine, &decision, &host_lc, port); + let destination_plan = decision + .endpoint + .destination + .as_ref() + .expect("destination plan hydrated"); // Defense-in-depth: resolve DNS and reject connections to internal IPs. let dns_connect_start = std::time::Instant::now(); - // The "non-empty" branch is the explicit-allowlist path; reading it first - // matches the policy decision narrative. - #[allow(clippy::if_not_else)] - let validated_addrs = if is_host_gateway_alias(&host_lc) - && let Some(gw) = *trusted_host_gateway + let connector = match validate_destination(DestinationRequest { + host: &host, + port, + sandbox_entrypoint_pid, + plan: destination_plan, + }) + .await { - // Trusted host-gateway path. The compute driver injected this hostname - // into /etc/hosts pointing at a known IP (read at proxy startup before - // user code runs). Bypass the normal SSRF tiers so link-local gateway - // addresses (used by rootless Podman with pasta) are not hard-blocked. - // Cloud metadata IPs and control-plane ports are still rejected. - match resolve_and_check_trusted_gateway(&host, port, gw, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: trusted-gateway check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: trusted-gateway check failed"), - ), - ) - .await?; - return Ok(()); - } - } - } else if !raw_allowed_ips.is_empty() { - // allowed_ips mode: validate resolved IPs against CIDR allowlist. - // Loopback and link-local are still always blocked. - match parse_allowed_ips(&raw_allowed_ips) { - Ok(nets) => { - match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) - .await - { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: allowed_ips check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "CONNECT {host_lc}:{port} blocked: allowed_ips check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: invalid allowed_ips in policy for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: invalid allowed_ips in policy"), - ), - ) - .await?; - return Ok(()); - } - } - } else if exact_declared_endpoint_host { - // Exact declared hostname mode: the operator explicitly allowed this - // host:port, so private IP resolution is permitted without duplicating - // the resolved IP in allowed_ips. Always-blocked addresses and - // control-plane ports remain denied. - match resolve_and_check_declared_endpoint(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: declared endpoint check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "CONNECT {host_lc}:{port} blocked: declared endpoint check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } else { - // Default: reject all internal IPs (loopback, RFC 1918, link-local). - match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: internal address {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: internal address"), - ), - ) - .await?; - return Ok(()); - } + Ok(connector) => connector, + Err(denial) => { + deny_connect_destination( + &mut client, + &denial, + peer_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + &decision, + &denial_tx, + &activity_tx, + ) + .await?; + return Ok(()); } }; @@ -1169,9 +1183,7 @@ async fn handle_tcp_connection( return Ok(()); } - let mut upstream = TcpStream::connect(validated_addrs.as_slice()) - .await - .into_diagnostic()?; + let mut upstream = connector.connect().await.into_diagnostic()?; debug!( "handle_tcp_connection dns_resolve_and_tcp_connect: {}ms host={host_lc}", @@ -1182,66 +1194,35 @@ async fn handle_tcp_connection( // Check if endpoint has L7 config for protocol-aware inspection, and // retain the generation for HTTP passthrough keep-alive tunnels. - let l7_route = query_l7_route_snapshot(&opa_engine, &decision, &host_lc, port); - let should_inspect_l7 = l7_inspection_active(l7_route.as_ref()); + hydrate_l7_route(&opa_engine, &mut decision); + let l7_route = decision.endpoint.l7_route.as_ref(); + let should_inspect_l7 = l7_inspection_active(l7_route); // Log the allowed CONNECT — use CONNECT_L7 when L7 inspection follows, // so log consumers can distinguish L4-only decisions from tunnel lifecycle events. - let connect_msg = if should_inspect_l7 { - "CONNECT_L7" - } else { - "CONNECT" - }; - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "opa") - .message(format!("{connect_msg} allowed {host_lc}:{port}")) - .build(); - ocsf_emit!(event); - } - emit_connect_activity_if_l4_only(&activity_tx, l7_route.as_ref()); + ocsf_emit!(build_connect_allow_ocsf_event( + peer_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + should_inspect_l7, + )); + emit_connect_activity_if_l4_only(&activity_tx, l7_route); // `effective_tls_skip` was resolved before the `200` above (the fail-closed // gate needs it) and drives the raw-tunnel branch below. - // Build L7 eval context (shared by TLS-terminated and plaintext paths). - let ctx = crate::l7::relay::L7EvalContext { - host: host_lc.clone(), - port, - policy_name: matched_policy.clone().unwrap_or_default(), - binary_path: decision - .binary - .as_ref() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(), - ancestors: decision - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - cmdline_paths: decision - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - secret_resolver: secret_resolver.clone(), - activity_tx: activity_tx.clone(), - dynamic_credentials: dynamic_credentials.clone(), - token_grant_resolver: dynamic_credentials - .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()), - }; + // Build request-processing context shared by CONNECT and forward HTTP. + let ctx = relay::http_context( + &decision, + secret_resolver.clone(), + activity_tx.clone(), + dynamic_credentials.clone(), + ); if effective_tls_skip { // Policy validation rejects fail-closed middleware overlapping @@ -1276,9 +1257,11 @@ async fn handle_tcp_connection( port = port, "tls: skip — bypassing TLS auto-detection, raw tunnel" ); - let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream) - .await - .into_diagnostic()?; + let Some(generation_guard) = relay::prepare_raw_relay(l7_route, &opa_engine, &decision) + else { + return Ok(()); + }; + relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await?; return Ok(()); } @@ -1298,61 +1281,13 @@ async fn handle_tcp_connection( let mut tls_upstream = crate::l7::tls::tls_connect_upstream(upstream, &host_lc, tls.upstream_config()) .await?; + let Some(relay_context) = + relay::prepare_http_relay(l7_route, &opa_engine, &decision, &ctx) + else { + return Ok(()); + }; - if let Some(route) = l7_route.as_ref().filter(|route| !route.configs.is_empty()) { - // L7 inspection on terminated TLS traffic. - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { - Ok(engine) => engine, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - if route.configs.len() == 1 { - crate::l7::relay::relay_with_inspection( - &route.configs[0].config, - tunnel_engine, - &mut tls_client, - &mut tls_upstream, - &ctx, - ) - .await - } else { - let configs: Vec = route - .configs - .iter() - .map(|snapshot| snapshot.config.clone()) - .collect(); - crate::l7::relay::relay_with_route_selection( - &configs, - tunnel_engine, - &mut tls_client, - &mut tls_upstream, - &ctx, - ) - .await - } - } else { - // No L7 config — relay with credential injection only. - let generation = l7_route - .as_ref() - .map_or(decision.generation, |route| route.generation); - let generation_guard = match opa_engine.generation_guard(generation) { - Ok(guard) => guard, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - crate::l7::relay::relay_passthrough_with_credentials( - &mut tls_client, - &mut tls_upstream, - &ctx, - &generation_guard, - Some(&opa_engine), - ) - .await - } + relay::relay_http_stream(&mut tls_client, &mut tls_upstream, relay_context).await }; if let Err(e) = tls_result.await { if is_benign_relay_error(&e) { @@ -1419,85 +1354,32 @@ async fn handle_tcp_connection( } } else if tunnel_protocol == TunnelProtocol::Http1 { // Plaintext HTTP detected. - if let Some(route) = l7_route.as_ref().filter(|route| !route.configs.is_empty()) { - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { - Ok(engine) => engine, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - let relay_result = if route.configs.len() == 1 { - crate::l7::relay::relay_with_inspection( - &route.configs[0].config, - tunnel_engine, - &mut client, - &mut upstream, - &ctx, - ) - .await - } else { - let configs: Vec = route - .configs - .iter() - .map(|snapshot| snapshot.config.clone()) - .collect(); - crate::l7::relay::relay_with_route_selection( - &configs, - tunnel_engine, - &mut client, - &mut upstream, - &ctx, - ) - .await - }; - if let Err(e) = relay_result { - if is_benign_relay_error(&e) { + let is_l7_relay = l7_route.is_some_and(|route| !route.configs.is_empty()); + let Some(relay_context) = relay::prepare_http_relay(l7_route, &opa_engine, &decision, &ctx) + else { + return Ok(()); + }; + if let Err(e) = relay::relay_http_stream(&mut client, &mut upstream, relay_context).await { + if is_benign_relay_error(&e) { + if is_l7_relay { debug!(host = %host_lc, port = port, error = %e, "L7 connection closed"); } else { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("L7 relay error: {e}")) - .build(); - ocsf_emit!(event); - } - } - } else { - // Plaintext HTTP, no L7 config — relay with credential injection. - let generation = l7_route - .as_ref() - .map_or(decision.generation, |route| route.generation); - let generation_guard = match opa_engine.generation_guard(generation) { - Ok(guard) => guard, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - if let Err(e) = crate::l7::relay::relay_passthrough_with_credentials( - &mut client, - &mut upstream, - &ctx, - &generation_guard, - Some(&opa_engine), - ) - .await - { - if is_benign_relay_error(&e) { debug!(host = %host_lc, port = port, error = %e, "HTTP relay closed"); - } else { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("HTTP relay error: {e}")) - .build(); - ocsf_emit!(event); } + } else { + let message = if is_l7_relay { + format!("L7 relay error: {e}") + } else { + format!("HTTP relay error: {e}") + }; + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .message(message) + .build(); + ocsf_emit!(event); } } } else { @@ -1560,9 +1442,11 @@ async fn handle_tcp_connection( port = port, "Non-TLS non-HTTP traffic detected, raw tunnel" ); - let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream) - .await - .into_diagnostic()?; + let Some(generation_guard) = relay::prepare_raw_relay(l7_route, &opa_engine, &decision) + else { + return Ok(()); + }; + relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await?; } Ok(()) @@ -1571,7 +1455,7 @@ async fn handle_tcp_connection( /// Resolved process identity for a TCP peer: binary path, PID, ancestor chain, /// cmdline paths, and the TOFU-verified binary hash. /// -/// Produced by [`resolve_process_identity`]; consumed by [`evaluate_opa_tcp`] +/// Produced by [`resolve_process_identity`]; consumed by [`authorize_egress_intent`] /// and by the identity-chain regression tests. #[cfg(target_os = "linux")] struct ResolvedIdentity { @@ -1605,7 +1489,7 @@ impl ResolvedIdentity { /// Error from [`resolve_process_identity`]. Carries the deny reason and /// whatever partial identity data was resolved before the failure so the -/// caller can include it in the [`ConnectDecision`] and OCSF event. +/// caller can include it in the [`EgressDecision`] and OCSF event. #[cfg(target_os = "linux")] struct IdentityError { reason: String, @@ -1702,7 +1586,7 @@ fn collect_ancestor_identities(start_pid: u32, stop_pid: u32) -> Vec<(u32, PathB /// walks each ancestor chain verifying every ancestor, and collects /// cmdline-derived absolute paths for script detection. /// -/// This is the identity-resolution block of [`evaluate_opa_tcp`] extracted +/// This is the identity-resolution block of [`authorize_egress_intent`] extracted /// into a standalone helper so it can be exercised by Linux-only regression /// tests without a full OPA engine. The key hot-swap invariant under test is /// that display paths are stripped for policy/logging, while integrity hashing @@ -1778,26 +1662,29 @@ fn resolve_process_identity( /// Evaluate OPA policy for a TCP connection with identity binding via /proc/net/tcp. #[cfg(target_os = "linux")] -fn evaluate_opa_tcp( +fn authorize_egress_intent( peer_addr: SocketAddr, engine: &OpaEngine, identity_cache: &BinaryIdentityCache, entrypoint_pid: &AtomicU32, - host: &str, - port: u16, -) -> ConnectDecision { + intent: EgressIntent, +) -> EgressDecision { use crate::opa::NetworkInput; use std::sync::atomic::Ordering; let deny = |reason: String, + identity: ProcessIdentityEvidence, binary: Option, binary_pid: Option, ancestors: Vec, cmdline_paths: Vec| - -> ConnectDecision { - ConnectDecision { + -> EgressDecision { + EgressDecision { + intent: intent.clone(), action: NetworkAction::Deny { reason }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), + identity, + endpoint: EndpointDecision::default(), binary, binary_pid, ancestors, @@ -1806,9 +1693,12 @@ fn evaluate_opa_tcp( }; if !crate::opa::network_binary_identity_required() { - let result = evaluate_endpoint_only_opa(engine, host, port); + let result = evaluate_endpoint_only_opa(engine, intent); debug!( - "evaluate_opa_tcp endpoint-only: host={host} port={port} action={:?}", + "authorize_egress_intent endpoint-only: host={} port={} transport={:?} action={:?}", + result.intent.destination.host, + result.intent.destination.port, + result.intent.transport, result.action ); return result; @@ -1818,6 +1708,7 @@ fn evaluate_opa_tcp( let Some(proc_net_anchor_pid) = proc_net_anchor_pid(entrypoint_pid) else { return deny( "entrypoint process not yet spawned".into(), + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), None, None, vec![], @@ -1833,6 +1724,7 @@ fn evaluate_opa_tcp( Err(err) => { return deny( err.reason, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), err.binary, err.binary_pid, err.ancestors, @@ -1850,8 +1742,8 @@ fn evaluate_opa_tcp( } = identity; let input = NetworkInput { - host: host.to_string(), - port, + host: intent.destination.host.clone(), + port: intent.destination.port, binary_path: bin_path.clone(), binary_sha256: bin_hash, ancestors: ancestors.clone(), @@ -1859,9 +1751,12 @@ fn evaluate_opa_tcp( }; let result = match engine.evaluate_network_action_with_generation(&input) { - Ok((action, generation)) => ConnectDecision { + Ok((action, generation)) => EgressDecision { + intent: intent.clone(), action, - generation, + l4_policy_generation: generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), binary: Some(bin_path), binary_pid: Some(binary_pid), ancestors, @@ -1869,6 +1764,7 @@ fn evaluate_opa_tcp( }, Err(e) => deny( format!("policy evaluation error: {e}"), + ProcessIdentityEvidence::Available, Some(bin_path), Some(binary_pid), ancestors, @@ -1876,8 +1772,11 @@ fn evaluate_opa_tcp( ), }; debug!( - "evaluate_opa_tcp TOTAL: {}ms host={host} port={port}", - total_start.elapsed().as_millis() + "authorize_egress_intent TOTAL: {}ms host={} port={} transport={:?}", + total_start.elapsed().as_millis(), + intent.destination.host, + intent.destination.port, + intent.transport, ); result } @@ -1896,10 +1795,10 @@ fn sidecar_topology_enabled() -> bool { .is_ok_and(|value| value == SIDECAR_SUPERVISOR_TOPOLOGY) } -fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> ConnectDecision { +fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> EgressDecision { let input = crate::opa::NetworkInput { - host: host.to_string(), - port, + host: intent.destination.host.clone(), + port: intent.destination.port, binary_path: PathBuf::new(), binary_sha256: String::new(), ancestors: vec![], @@ -1907,19 +1806,29 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> Conn }; match engine.evaluate_network_action_with_generation(&input) { - Ok((action, generation)) => ConnectDecision { + Ok((action, generation)) => EgressDecision { + intent, action, - generation, + l4_policy_generation: generation, + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::EndpointOnlyMode, + ), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], cmdline_paths: vec![], }, - Err(e) => ConnectDecision { + Err(e) => EgressDecision { + intent, action: NetworkAction::Deny { reason: format!("policy evaluation error: {e}"), }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::EndpointOnlyMode, + ), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], @@ -1930,23 +1839,27 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> Conn /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] -fn evaluate_opa_tcp( +fn authorize_egress_intent( _peer_addr: SocketAddr, engine: &OpaEngine, _identity_cache: &BinaryIdentityCache, _entrypoint_pid: &AtomicU32, - host: &str, - port: u16, -) -> ConnectDecision { + intent: EgressIntent, +) -> EgressDecision { if !crate::opa::network_binary_identity_required() { - return evaluate_endpoint_only_opa(engine, host, port); + return evaluate_endpoint_only_opa(engine, intent); } - ConnectDecision { + EgressDecision { + intent, action: NetworkAction::Deny { reason: "identity binding unavailable on this platform".into(), }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::UnsupportedPlatform, + ), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], @@ -2388,17 +2301,6 @@ async fn write_all(writer: &mut (impl tokio::io::AsyncWrite + Unpin), data: &[u8 Ok(()) } -#[derive(Debug, Clone)] -struct L7ConfigSnapshot { - config: crate::l7::L7EndpointConfig, -} - -#[derive(Debug, Clone)] -struct L7RouteSnapshot { - configs: Vec, - generation: u64, -} - fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette::Report) { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) @@ -2414,13 +2316,45 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } -/// Query L7 endpoint config from the OPA engine for a matched CONNECT decision. +/// Query L7 endpoint config from the OPA engine for an allowed egress decision. /// /// Returns `Some(L7EndpointConfig)` if the matched endpoint has L7 config (protocol field), /// `None` for L4-only endpoints. +fn hydrate_l7_route(engine: &OpaEngine, decision: &mut EgressDecision) { + let host = decision.intent.destination.host.clone(); + let port = decision.intent.destination.port; + decision.endpoint.l7_route = query_l7_route_snapshot(engine, decision, &host, port); +} + +fn hydrate_tls_mode(engine: &OpaEngine, decision: &mut EgressDecision) { + let host = decision.intent.destination.host.clone(); + let port = decision.intent.destination.port; + decision.endpoint.tls_mode = query_tls_mode(engine, decision, &host, port); +} + +fn hydrate_destination_plan( + engine: &OpaEngine, + decision: &mut EgressDecision, + trusted_host_gateway: Option, +) -> std::result::Result<(), DestinationDenial> { + let host = decision.intent.destination.host.clone(); + let port = decision.intent.destination.port; + let raw_allowed_ips = query_allowed_ips(engine, decision, &host, port); + let exact_declared_host = query_exact_declared_endpoint_host(engine, decision, &host, port); + let plan = build_validation_plan( + &host, + &host.to_ascii_lowercase(), + trusted_host_gateway, + &raw_allowed_ips, + exact_declared_host, + )?; + decision.endpoint.destination = Some(plan); + Ok(()) +} + fn query_l7_route_snapshot( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> Option { @@ -2458,7 +2392,7 @@ fn query_l7_route_snapshot( ); Some(L7RouteSnapshot { configs, - generation, + l7_policy_generation: generation, }) } Err(e) => { @@ -2490,7 +2424,7 @@ fn select_l7_config_for_path<'a>( /// This extracts `tls: skip` from the endpoint even when no `protocol` is set. fn query_tls_mode( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> crate::l7::TlsMode { @@ -3030,7 +2964,7 @@ fn parse_allowed_ips(raw: &[String]) -> std::result::Result, S /// Query `allowed_ips` from the matched endpoint config for a CONNECT decision. fn query_allowed_ips( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> Vec { @@ -3073,7 +3007,7 @@ fn query_allowed_ips( /// Query whether the matched endpoint was declared as this exact hostname. fn query_exact_declared_endpoint_host( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> bool { @@ -3413,9 +3347,13 @@ fn rewrite_forward_request( output.extend_from_slice(b"\r\n"); let rewritten_header_end = output.len(); - // Append any overflow body bytes from the original buffer + // Append only bytes that belong to the first request body. The initial + // proxy read can also contain a pipelined follow-on request; forwarding + // that as body overflow would bypass its own policy evaluation. if header_end < used { - output.extend_from_slice(&raw[header_end..used]); + let overflow = &raw[header_end..used]; + let body_prefix_len = initial_forward_body_prefix_len(&header_str, overflow); + output.extend_from_slice(&overflow[..body_prefix_len]); } // Fail-closed: scan for any remaining unresolved placeholders @@ -3436,6 +3374,66 @@ fn rewrite_forward_request( Ok(output) } +fn initial_forward_body_prefix_len(header_str: &str, overflow: &[u8]) -> usize { + match crate::l7::rest::parse_body_length(header_str) { + Ok(crate::l7::provider::BodyLength::None) => 0, + Ok(crate::l7::provider::BodyLength::ContentLength(len)) => usize::try_from(len) + .unwrap_or(usize::MAX) + .min(overflow.len()), + Ok(crate::l7::provider::BodyLength::Chunked) => { + complete_chunked_body_prefix_len(overflow).unwrap_or(overflow.len()) + } + // Invalid framing is rejected by the guarded relay before an upstream + // body write. Keep the bytes available so that parser sees the same + // malformed request instead of blocking while trying to re-read them. + Err(_) => overflow.len(), + } +} + +/// Return the complete chunked body length when its terminator is already in +/// the initial read. `None` means more body bytes are required. +fn complete_chunked_body_prefix_len(bytes: &[u8]) -> Option { + let mut pos = 0usize; + loop { + let line_end = bytes[pos..] + .windows(2) + .position(|window| window == b"\r\n")? + + pos; + let size_line = std::str::from_utf8(&bytes[pos..line_end]).ok()?; + let size = usize::from_str_radix( + size_line + .split(';') + .next() + .map(str::trim) + .unwrap_or_default(), + 16, + ) + .ok()?; + pos = line_end.checked_add(2)?; + + if size == 0 { + loop { + let trailer_end = bytes[pos..] + .windows(2) + .position(|window| window == b"\r\n")? + + pos; + let empty = trailer_end == pos; + pos = trailer_end.checked_add(2)?; + if empty { + return Some(pos); + } + } + } + + let chunk_end = pos.checked_add(size)?; + let framed_end = chunk_end.checked_add(2)?; + if framed_end > bytes.len() || &bytes[chunk_end..framed_end] != b"\r\n" { + return None; + } + pos = framed_end; + } +} + struct ForwardRelayOptions<'a> { generation_guard: &'a PolicyGenerationGuard, websocket_extensions: crate::l7::rest::WebSocketExtensionMode, @@ -3644,20 +3642,19 @@ async fn handle_forward_proxy( let opa_clone = opa_engine.clone(); let cache_clone = identity_cache.clone(); let pid_clone = entrypoint_pid.clone(); - let host_clone = host_lc.clone(); - let decision = tokio::task::spawn_blocking(move || { - evaluate_opa_tcp( - peer_addr, - &opa_clone, - &cache_clone, - &pid_clone, - &host_clone, - port, - ) + let intent = EgressIntent::forward_http(host_lc.clone(), port); + let mut decision = tokio::task::spawn_blocking(move || { + authorize_egress_intent(peer_addr, &opa_clone, &cache_clone, &pid_clone, intent) }) .await .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + debug!( + transport = ?decision.intent.transport, + identity = ?decision.identity, + "Authorized explicit proxy egress intent" + ); + // Build log context let binary_str = decision .binary @@ -3691,28 +3688,18 @@ async fn handle_forward_proxy( let matched_policy = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone(), NetworkAction::Deny { reason } => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "opa") - .message(format!("FORWARD denied {method} {host_lc}:{port}{path}")) - .build(); - ocsf_emit!(event); - } + ocsf_emit!(build_forward_policy_deny_ocsf_event( + peer_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + reason, + )); emit_denial_simple( denial_tx, &host_lc, @@ -3743,19 +3730,22 @@ async fn handle_forward_proxy( binary = %binary_str, binary_pid = %pid_str, matched_policy = %policy_str, - decision_generation = decision.generation, + l4_policy_generation = decision.l4_policy_generation, current_generation = opa_engine.current_generation(), action = ?decision.action, "Forward proxy L4 policy decision" ); let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - let forward_generation_guard = match opa_engine.generation_guard(decision.generation) { + let forward_generation_guard = match relay::pin_policy_generation( + &opa_engine, + decision.l4_policy_generation, + ) { Ok(guard) => guard, Err(e) => { warn!( host = %host_lc, port, - decision_generation = decision.generation, + l4_policy_generation = decision.l4_policy_generation, current_generation = opa_engine.current_generation(), error = %e, "Forward proxy rejected request because policy generation changed after L4 decision" @@ -3783,53 +3773,39 @@ async fn handle_forward_proxy( // same policy inputs before it is forwarded. let mut forward_l7_reeval: Option<(crate::l7::L7EndpointConfig, crate::l7::L7RequestInfo)> = None; - let mut forward_upgrade_config: Option = None; - let mut forward_upgrade_target = String::new(); - let mut forward_upgrade_query_params = std::collections::HashMap::new(); - let mut forward_websocket_request = - crate::l7::rest::request_is_websocket_upgrade(&forward_request_bytes); - let mut request_body_credential_rewrite = false; - let l7_ctx = crate::l7::relay::L7EvalContext { - host: host_lc.clone(), - port, - policy_name: matched_policy.clone().unwrap_or_default(), - binary_path: decision - .binary - .as_ref() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(), - ancestors: decision - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - cmdline_paths: decision - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - secret_resolver: secret_resolver.clone(), - activity_tx: activity_tx.cloned(), - dynamic_credentials: dynamic_credentials.clone(), - token_grant_resolver: dynamic_credentials - .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()), - }; + let mut forward_upgrade_config: Option = None; + let mut forward_upgrade_target = String::new(); + let mut forward_upgrade_query_params = std::collections::HashMap::new(); + let mut forward_websocket_request = + crate::l7::rest::request_is_websocket_upgrade(&forward_request_bytes); + let mut request_body_credential_rewrite = false; + let l7_ctx = relay::http_context( + &decision, + secret_resolver.clone(), + activity_tx.cloned(), + dynamic_credentials.clone(), + ); let mut l7_activity_pending = false; // 4b. If the endpoint has L7 config, evaluate the request against - // L7 policy. The forward proxy handles exactly one request per - // connection (Connection: close), so a single evaluation suffices. - if let Some(route) = query_l7_route_snapshot(&opa_engine, &decision, &host_lc, port) - && !route.configs.is_empty() + // L7 policy. The forward proxy handles exactly one request per + // connection, so a single evaluation suffices. The shared HTTP relay + // strips hop-by-hop `Connection` headers and drops the upstream after + // the response instead of asking the upstream to close it. + hydrate_l7_route(&opa_engine, &mut decision); + if let Some(route) = decision + .endpoint + .l7_route + .as_ref() + .filter(|route| !route.configs.is_empty()) { - if route.generation != forward_generation_guard.captured_generation() { + if route.l7_policy_generation != forward_generation_guard.captured_generation() { warn!( host = %host_lc, port, - decision_generation = decision.generation, - guard_generation = forward_generation_guard.captured_generation(), - route_generation = route.generation, + l4_policy_generation = decision.l4_policy_generation, + l4_guard_generation = forward_generation_guard.captured_generation(), + l7_policy_generation = route.l7_policy_generation, current_generation = opa_engine.current_generation(), "Forward proxy rejected request because L7 route lookup used a different policy generation" ); @@ -3839,7 +3815,7 @@ async fn handle_forward_proxy( miette::miette!( "policy changed before forward L7 evaluation [expected_generation:{} current_generation:{}]", forward_generation_guard.captured_generation(), - route.generation, + route.l7_policy_generation, ), ); emit_activity_simple(activity_tx, true, "policy_stale"); @@ -3855,13 +3831,13 @@ async fn handle_forward_proxy( .await?; return Ok(()); } - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { + let tunnel_engine = match relay::pin_l7_evaluator(&opa_engine, route.l7_policy_generation) { Ok(engine) => engine, Err(e) => { warn!( host = %host_lc, port, - route_generation = route.generation, + l7_policy_generation = route.l7_policy_generation, current_generation = opa_engine.current_generation(), error = %e, "Forward proxy rejected request because L7 tunnel engine could not be cloned" @@ -4258,290 +4234,65 @@ async fn handle_forward_proxy( // - Otherwise: reject internal IPs, allow public IPs through. // When the policy host is already a literal IP address, treat it as // implicitly allowed — the user explicitly declared the destination. - let mut raw_allowed_ips = query_allowed_ips(&opa_engine, &decision, &host_lc, port); - if raw_allowed_ips.is_empty() { - raw_allowed_ips = implicit_allowed_ips_for_ip_host(&host); - } - let exact_declared_endpoint_host = - query_exact_declared_endpoint_host(&opa_engine, &decision, &host_lc, port); - - // The trusted-gateway branch is the first path; reading it before the - // allowed_ips and default branches matches the policy decision narrative. - #[allow(clippy::if_not_else)] - let addrs = if is_host_gateway_alias(&host_lc) - && let Some(gw) = *trusted_host_gateway - { - // Trusted host-gateway path. Mirrors the CONNECT path logic. - match resolve_and_check_trusted_gateway(&host, port, gw, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: trusted-gateway check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("{method} {host_lc}:{port} blocked: trusted-gateway check failed"), - ), - ) - .await?; - return Ok(()); - } - } - } else if !raw_allowed_ips.is_empty() { - // allowed_ips mode: validate resolved IPs against CIDR allowlist. - match parse_allowed_ips(&raw_allowed_ips) { - Ok(nets) => { - match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) - .await - { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: allowed_ips check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: allowed_ips check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: invalid allowed_ips in policy for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: invalid allowed_ips in policy" - ), - ), - ) - .await?; - return Ok(()); - } - } - } else if exact_declared_endpoint_host { - // Exact declared hostname mode mirrors CONNECT: private resolved - // addresses are allowed for this operator-declared host:port, while - // always-blocked addresses and control-plane ports remain denied. - match resolve_and_check_declared_endpoint(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: declared endpoint check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: declared endpoint check failed" - ), - ), - ) - .await?; - return Ok(()); - } + match hydrate_destination_plan(&opa_engine, &mut decision, *trusted_host_gateway) { + Ok(()) => {} + Err(denial) => { + deny_forward_destination( + client, + &denial, + peer_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + &decision, + denial_tx, + activity_tx, + ) + .await?; + return Ok(()); } - } else { - // No allowed_ips: reject internal IPs, allow public IPs through. - match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: internal IP without allowed_ips for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("{method} {host_lc}:{port} blocked: internal address"), - ), - ) - .await?; - return Ok(()); - } + } + let destination_plan = decision + .endpoint + .destination + .as_ref() + .expect("destination plan hydrated"); + + let connector = match validate_destination(DestinationRequest { + host: &host, + port, + sandbox_entrypoint_pid, + plan: destination_plan, + }) + .await + { + Ok(connector) => connector, + Err(denial) => { + deny_forward_destination( + client, + &denial, + peer_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + &decision, + denial_tx, + activity_tx, + ) + .await?; + return Ok(()); } }; @@ -4570,7 +4321,7 @@ async fn handle_forward_proxy( } // 6. Connect upstream - let mut upstream = match TcpStream::connect(addrs.as_slice()).await { + let mut upstream = match connector.connect().await { Ok(s) => s, Err(e) => { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -4675,7 +4426,6 @@ async fn handle_forward_proxy( } }; } - forward_request_bytes = match inject_token_grant_for_forward_request( method, &upstream_target, @@ -4777,28 +4527,18 @@ async fn handle_forward_proxy( // The request has now survived middleware, token grant, credential // rewriting, generation checks, and the HTTP relay. Only now record the // final allowed outcome. - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "opa") - .message(format!("FORWARD allowed {method} {host_lc}:{port}{path}")) - .build(); - ocsf_emit!(event); - } + ocsf_emit!(build_forward_allow_ocsf_event( + peer_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + )); emit_forward_success_activity(activity_tx, l7_activity_pending); if let crate::l7::provider::RelayOutcome::Upgraded { @@ -5127,6 +4867,28 @@ network_policies: {} assert!(body.get("agent_guidance").is_none()); } + #[test] + fn forward_policy_denial_ocsf_includes_validation_rationale() { + let reason = "policy validation failed; fail-closed quarantine is active; candidate version 7 rejected: conflicting tls metadata"; + let event = build_forward_policy_deny_ocsf_event( + "127.0.0.1:45123".parse().unwrap(), + "GET", + "api.example.com", + 80, + "/v1/models", + "/usr/bin/curl", + "42", + "/usr/bin/bash", + "curl http://api.example.com/v1/models", + reason, + ); + let json = event.to_json().unwrap(); + + assert_eq!(json["status_detail"], reason); + assert_eq!(json["action"], "Denied"); + assert_eq!(json["disposition"], "Blocked"); + } + #[test] fn endpoint_only_opa_allows_declared_endpoint_without_process_identity() { let policy = include_str!("../data/sandbox-policy.rego"); @@ -5150,7 +4912,10 @@ network_policies: let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) .expect("relaxed engine"); - let decision = evaluate_endpoint_only_opa(&engine, "host.k3d.internal", 56123); + let decision = evaluate_endpoint_only_opa( + &engine, + EgressIntent::connect("host.k3d.internal".to_string(), 56123), + ); assert_eq!( decision.action, NetworkAction::Allow { @@ -5160,7 +4925,10 @@ network_policies: assert!(decision.binary.is_none()); assert!(decision.ancestors.is_empty()); - let denied = evaluate_endpoint_only_opa(&engine, "api.example.com", 443); + let denied = evaluate_endpoint_only_opa( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + ); assert!( matches!(denied.action, NetworkAction::Deny { .. }), "endpoint-only mode must still deny undeclared endpoints" @@ -5350,11 +5118,11 @@ network_policies: configs: vec![L7ConfigSnapshot { config: websocket_l7_config(crate::l7::L7Protocol::Rest, false), }], - generation: 1, + l7_policy_generation: 1, }; let l4_route = L7RouteSnapshot { configs: Vec::new(), - generation: 1, + l7_policy_generation: 1, }; emit_connect_activity_if_l4_only(&activity_tx, Some(&l7_route)); @@ -5762,11 +5530,14 @@ network_policies: ) { let policy = include_str!("../data/sandbox-policy.rego"); let engine = OpaEngine::from_strings(policy, data).unwrap(); - let decision = ConnectDecision { + let decision = EgressDecision { + intent: EgressIntent::forward_http(host.to_string(), port), action: NetworkAction::Allow { matched_policy: Some(policy_name.to_string()), }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), binary: Some(PathBuf::from("/usr/bin/node")), binary_pid: None, ancestors: vec![], @@ -5779,7 +5550,7 @@ network_policies: .config .clone(); let tunnel_engine = engine - .clone_engine_for_tunnel(route.generation) + .clone_engine_for_tunnel(route.l7_policy_generation) .expect("tunnel engine"); let ctx = crate::l7::relay::L7EvalContext { host: host.to_string(), @@ -9267,7 +9038,7 @@ network_policies: /// itself), binds to `current_exe()`, and never falls through to the /// whole-`/proc` scan — the environment-sensitive path that made a forked /// child flaky under a busy CI `/proc`. Callers gate on Linux; - /// `evaluate_opa_tcp` denies unconditionally without `/proc`. + /// `authorize_egress_intent` denies unconditionally without `/proc`. async fn drive_connect_through_handler( endpoint_yaml: &str, connect_target: &str, @@ -9343,6 +9114,66 @@ network_policies: (completed, stdout, denial_stages) } + /// Drives an absolute-form request through the same explicit-proxy entry + /// point used by CONNECT and returns the response and denial stages. + async fn drive_forward_through_handler( + endpoint_yaml: &str, + target: &str, + ) -> (Vec, Vec) { + const POLICY_REGO: &str = include_str!("../data/sandbox-policy.rego"); + + let exe = std::env::current_exe().expect("current_exe"); + let data = format!( + r#"network_policies: + test_allow: + name: test_allow + endpoints: +{endpoint_yaml} binaries: + - {{ path: "{exe}" }} +"#, + exe = exe.display(), + ); + let engine = Arc::new(OpaEngine::from_strings(POLICY_REGO, &data).expect("load policy")); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_port = listener.local_addr().unwrap().port(); + let target = target.to_string(); + let client = tokio::spawn(async move { + let mut socket = TcpStream::connect(("127.0.0.1", proxy_port)).await.unwrap(); + let request = format!("GET {target} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + socket.write_all(request.as_bytes()).await.unwrap(); + let mut response = Vec::new(); + socket.read_to_end(&mut response).await.unwrap(); + response + }); + + let (server, _) = listener.accept().await.unwrap(); + let (denial_tx, mut denial_rx) = mpsc::unbounded_channel(); + Box::pin(handle_tcp_connection( + server, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(std::process::id())), + None, + None, + None, + Arc::new(None), + None, + None, + Some(denial_tx), + None, + )) + .await + .expect("forward handler should complete"); + + let response = client.await.expect("client task"); + let mut denial_stages = Vec::new(); + while let Ok(event) = denial_rx.try_recv() { + denial_stages.push(event.denial_stage); + } + (response, denial_stages) + } + /// End-to-end regression for the gator finding on PR #2162: with no TLS /// termination state, a terminating `CONNECT` must have its 503 written as /// the FIRST bytes on the socket — never after a `200 Connection @@ -9414,6 +9245,29 @@ network_policies: ); } + #[tokio::test] + async fn forward_handler_preserves_ssrf_response_and_denial_stage() { + if !cfg!(target_os = "linux") { + eprintln!("skipping: handler identity binding requires /proc (Linux)"); + return; + } + + let (response, denial_stages) = Box::pin(drive_forward_through_handler( + " - { host: \"127.0.0.1\", port: 80 }\n", + "http://127.0.0.1/private", + )) + .await; + + let response = String::from_utf8_lossy(&response); + assert!( + response.starts_with("HTTP/1.1 403 Forbidden"), + "internal forward destination must get the SSRF 403; got: {response:?}" + ); + assert!(response.contains("ssrf_denied")); + assert!(response.contains("GET 127.0.0.1:80 blocked: allowed_ips check failed")); + assert_eq!(denial_stages, ["ssrf"]); + } + /// A real `tls: skip` policy path through the handler is exempt from the /// fail-closed gate even with no TLS termination state: the handler proceeds /// past the refusal to the raw-tunnel upstream connect (which stalls on the @@ -9492,9 +9346,12 @@ network_policies: panic!("glob binary must be allowed, got deny: {reason}") } } - let decision = ConnectDecision { + let decision = EgressDecision { + intent: EgressIntent::connect("203.0.113.10".to_string(), 443), action, - generation, + l4_policy_generation: generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), binary: Some(input.binary_path), binary_pid: Some(1), ancestors: vec![], @@ -9827,4 +9684,7 @@ network_policies: } } } + + #[path = "compatibility.rs"] + mod compatibility; } diff --git a/crates/openshell-supervisor-network/src/proxy/destination.rs b/crates/openshell-supervisor-network/src/proxy/destination.rs new file mode 100644 index 0000000000..a260d477f6 --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/destination.rs @@ -0,0 +1,299 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared external destination validation and upstream dial boundary. + +use super::{ + implicit_allowed_ips_for_ip_host, is_host_gateway_alias, parse_allowed_ips, + resolve_and_check_allowed_ips, resolve_and_check_declared_endpoint, + resolve_and_check_trusted_gateway, resolve_and_reject_internal, +}; +use ipnet::IpNet; +use std::net::{IpAddr, SocketAddr}; +use tokio::net::TcpStream; + +/// Address-validation mode selected from the current endpoint configuration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum AddressAuthorization { + DefaultPublicOnly, + ExplicitAllowedIps(Vec), + ExactDeclaredHost, + ImplicitIpLiteral(IpAddr), + TrustedGatewayAlias { expected_ip: IpAddr }, +} + +/// Fully materialized input to shared destination validation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct DestinationValidationPlan { + pub(super) address_authorization: AddressAuthorization, +} + +/// Inputs needed to apply the current SSRF and endpoint destination policy. +pub(super) struct DestinationRequest<'a> { + pub(super) host: &'a str, + pub(super) port: u16, + pub(super) sandbox_entrypoint_pid: u32, + pub(super) plan: &'a DestinationValidationPlan, +} + +/// Destination-validation branch that rejected an egress request. +/// +/// Adapters use this classification to preserve their existing HTTP response +/// and OCSF message shapes while sharing the underlying validation logic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DestinationDenialKind { + TrustedGateway, + InvalidAllowedIps, + AllowedIps, + DeclaredEndpoint, + InternalAddress, +} + +#[derive(Debug)] +pub(super) struct DestinationDenial { + pub(super) kind: DestinationDenialKind, + pub(super) reason: String, +} + +impl DestinationDenial { + fn new(kind: DestinationDenialKind, reason: String) -> Self { + Self { kind, reason } + } +} + +/// Select one current destination-validation mode without changing precedence. +pub(super) fn build_validation_plan( + host: &str, + normalized_host: &str, + trusted_host_gateway: Option, + raw_allowed_ips: &[String], + exact_declared_endpoint_host: bool, +) -> Result { + let address_authorization = if is_host_gateway_alias(normalized_host) + && let Some(expected_ip) = trusted_host_gateway + { + AddressAuthorization::TrustedGatewayAlias { expected_ip } + } else if !raw_allowed_ips.is_empty() { + AddressAuthorization::ExplicitAllowedIps(parse_allowed_ips(raw_allowed_ips).map_err( + |reason| DestinationDenial::new(DestinationDenialKind::InvalidAllowedIps, reason), + )?) + } else if let Some(ip) = implicit_allowed_ips_for_ip_host(host) + .first() + .and_then(|raw| raw.parse::().ok()) + { + AddressAuthorization::ImplicitIpLiteral(ip) + } else if exact_declared_endpoint_host { + AddressAuthorization::ExactDeclaredHost + } else { + AddressAuthorization::DefaultPublicOnly + }; + + Ok(DestinationValidationPlan { + address_authorization, + }) +} + +/// Validated, but not yet opened, upstream destination. +/// +/// The explicit proxy adapter controls when `connect` is called so CONNECT and +/// forward HTTP retain their current upstream-dial timing during the refactor. +pub(super) struct UpstreamConnector { + host: String, + port: u16, + addrs: Vec, +} + +impl UpstreamConnector { + pub(super) async fn connect(&self) -> std::io::Result { + tracing::debug!( + host = %self.host, + port = self.port, + address_count = self.addrs.len(), + "Opening validated upstream connection" + ); + TcpStream::connect(self.addrs.as_slice()).await + } + + fn new(host: &str, port: u16, addrs: Vec) -> Self { + Self { + host: host.to_string(), + port, + addrs, + } + } +} + +/// Resolve and validate a destination using the existing proxy security rules. +pub(super) async fn validate_destination( + request: DestinationRequest<'_>, +) -> Result { + let DestinationRequest { + host, + port, + sandbox_entrypoint_pid, + plan, + } = request; + + let addrs = match &plan.address_authorization { + AddressAuthorization::TrustedGatewayAlias { expected_ip } => { + resolve_and_check_trusted_gateway(host, port, *expected_ip, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::TrustedGateway, reason) + })? + } + AddressAuthorization::ExplicitAllowedIps(networks) => { + resolve_and_check_allowed_ips(host, port, networks, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::AllowedIps, reason) + })? + } + AddressAuthorization::ImplicitIpLiteral(ip) => { + let network = IpNet::from(*ip); + resolve_and_check_allowed_ips(host, port, &[network], sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::AllowedIps, reason) + })? + } + AddressAuthorization::ExactDeclaredHost => { + resolve_and_check_declared_endpoint(host, port, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::DeclaredEndpoint, reason) + })? + } + AddressAuthorization::DefaultPublicOnly => { + resolve_and_reject_internal(host, port, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::InternalAddress, reason) + })? + } + }; + + Ok(UpstreamConnector::new(host, port, addrs)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, Ipv4Addr}; + + fn request<'a>(host: &'a str, plan: &'a DestinationValidationPlan) -> DestinationRequest<'a> { + DestinationRequest { + host, + port: 80, + sandbox_entrypoint_pid: 0, + plan, + } + } + + #[tokio::test] + async fn default_mode_classifies_loopback_as_internal_address() { + let plan = DestinationValidationPlan { + address_authorization: AddressAuthorization::DefaultPublicOnly, + }; + let denial = validate_destination(request("127.0.0.1", &plan)) + .await + .err() + .expect("loopback must be denied"); + + assert_eq!(denial.kind, DestinationDenialKind::InternalAddress); + } + + #[tokio::test] + async fn invalid_allowed_ips_has_a_distinct_denial_kind() { + let denial = build_validation_plan( + "api.example.test", + "api.example.test", + None, + &["not-an-ip".to_string()], + false, + ) + .expect_err("invalid allowed_ips must be denied"); + + assert_eq!(denial.kind, DestinationDenialKind::InvalidAllowedIps); + } + + #[tokio::test] + async fn declared_endpoint_preserves_its_denial_classification() { + let plan = DestinationValidationPlan { + address_authorization: AddressAuthorization::ExactDeclaredHost, + }; + let denial = validate_destination(request("127.0.0.1", &plan)) + .await + .err() + .expect("declared loopback must remain denied"); + + assert_eq!(denial.kind, DestinationDenialKind::DeclaredEndpoint); + } + + #[tokio::test] + async fn trusted_gateway_preserves_its_denial_classification() { + let plan = DestinationValidationPlan { + address_authorization: AddressAuthorization::TrustedGatewayAlias { + expected_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + }, + }; + let denial = validate_destination(request("host.openshell.internal", &plan)) + .await + .err() + .expect("loopback cannot be a trusted gateway"); + + assert_eq!(denial.kind, DestinationDenialKind::TrustedGateway); + } + + #[test] + fn validation_mode_precedence_is_explicit_and_stable() { + let trusted_ip = IpAddr::V4(Ipv4Addr::new(169, 254, 1, 2)); + let trusted = build_validation_plan( + "host.openshell.internal", + "host.openshell.internal", + Some(trusted_ip), + &["10.0.0.0/8".to_string()], + true, + ) + .unwrap(); + assert_eq!( + trusted.address_authorization, + AddressAuthorization::TrustedGatewayAlias { + expected_ip: trusted_ip + } + ); + + let explicit = build_validation_plan( + "10.2.3.4", + "10.2.3.4", + None, + &["10.0.0.0/8".to_string()], + true, + ) + .unwrap(); + assert_eq!( + explicit.address_authorization, + AddressAuthorization::ExplicitAllowedIps(vec!["10.0.0.0/8".parse().unwrap()]) + ); + + let implicit = build_validation_plan("10.2.3.4", "10.2.3.4", None, &[], true).unwrap(); + assert_eq!( + implicit.address_authorization, + AddressAuthorization::ImplicitIpLiteral("10.2.3.4".parse().unwrap()) + ); + + let declared = + build_validation_plan("private.example", "private.example", None, &[], true).unwrap(); + assert_eq!( + declared.address_authorization, + AddressAuthorization::ExactDeclaredHost + ); + + let default = + build_validation_plan("*.example.com", "*.example.com", None, &[], false).unwrap(); + assert_eq!( + default.address_authorization, + AddressAuthorization::DefaultPublicOnly + ); + } +} diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs new file mode 100644 index 0000000000..bf1b05131b --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Transport-neutral egress inputs and authorization results. +//! +//! Explicit proxy adapters normalize their protocol-specific request into an +//! [`EgressIntent`]. Authorization then returns an [`EgressDecision`] that is +//! consumed by destination validation and relay selection. Keeping these types +//! independent of CONNECT and forward HTTP prevents policy behavior from +//! drifting as more adapters are added. + +use super::destination::DestinationValidationPlan; +use crate::opa::NetworkAction; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub(super) struct L7ConfigSnapshot { + pub(super) config: crate::l7::L7EndpointConfig, +} + +#[derive(Debug, Clone)] +pub(super) struct L7RouteSnapshot { + pub(super) configs: Vec, + /// Policy generation used to materialize this L7 route. + pub(super) l7_policy_generation: u64, +} + +/// Endpoint metadata materialized for an allowed egress decision. +/// +/// The migration hydrates these fields at the same points the legacy handlers +/// queried them so policy-reload and upstream-connect timing remain unchanged. +#[derive(Debug, Clone)] +pub(super) struct EndpointDecision { + pub(super) tls_mode: crate::l7::TlsMode, + pub(super) l7_route: Option, + /// Destination authorization selected at the legacy hydration point. + pub(super) destination: Option, +} + +impl Default for EndpointDecision { + fn default() -> Self { + Self { + tls_mode: crate::l7::TlsMode::Auto, + l7_route: None, + destination: None, + } + } +} + +/// Userland surface through which an external egress request arrived. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum EgressTransport { + Connect, + ForwardHttp, +} + +/// Destination requested by an explicit proxy adapter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RequestedDestination { + pub(super) host: String, + pub(super) port: u16, +} + +/// Transport-neutral description of an external egress request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct EgressIntent { + pub(super) transport: EgressTransport, + pub(super) destination: RequestedDestination, +} + +impl EgressIntent { + pub(super) fn connect(host: String, port: u16) -> Self { + Self::new(EgressTransport::Connect, host, port) + } + + pub(super) fn forward_http(host: String, port: u16) -> Self { + Self::new(EgressTransport::ForwardHttp, host, port) + } + + fn new(transport: EgressTransport, host: String, port: u16) -> Self { + Self { + transport, + destination: RequestedDestination { host, port }, + } + } +} + +/// Why process identity is absent from an egress decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub(super) enum IdentityUnavailableReason { + EndpointOnlyMode, + LookupFailed, + UnsupportedPlatform, +} + +/// Process evidence captured for policy evaluation and audit logging. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub(super) enum ProcessIdentityEvidence { + Available, + Unavailable(IdentityUnavailableReason), +} + +/// Result of authorizing a normalized egress intent. +/// +/// The identity fields intentionally mirror the former CONNECT-specific +/// decision during the compatibility migration. Endpoint configuration is +/// hydrated at the legacy query points without changing lookup precedence or +/// failure defaults. +pub(super) struct EgressDecision { + pub(super) intent: EgressIntent, + pub(super) action: NetworkAction, + /// Policy generation used for the L4 network decision. + pub(super) l4_policy_generation: u64, + /// Whether process identity evidence was available to policy evaluation. + pub(super) identity: ProcessIdentityEvidence, + /// Endpoint behavior hydrated for destination validation and relays. + pub(super) endpoint: EndpointDecision, + /// Resolved binary path. + pub(super) binary: Option, + /// PID owning the socket. + pub(super) binary_pid: Option, + /// Ancestor binary paths from process tree walk. + pub(super) ancestors: Vec, + /// Cmdline-derived absolute paths (for script detection). + pub(super) cmdline_paths: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn adapters_create_transport_specific_intents() { + let connect = EgressIntent::connect("api.example.com".to_string(), 443); + let forward = EgressIntent::forward_http("api.example.com".to_string(), 80); + + assert_eq!(connect.transport, EgressTransport::Connect); + assert_eq!(connect.destination.host, "api.example.com"); + assert_eq!(connect.destination.port, 443); + assert_eq!(forward.transport, EgressTransport::ForwardHttp); + assert_eq!(forward.destination.port, 80); + } +} diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs new file mode 100644 index 0000000000..50a3b3c26a --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -0,0 +1,393 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared relay primitives for authorized explicit-proxy egress. + +use super::{EgressDecision, L7RouteSnapshot, emit_l7_tunnel_close_after_policy_change}; +use crate::l7::relay::L7EvalContext; +use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard, TunnelPolicyEngine}; +use miette::{IntoDiagnostic, Result}; +use openshell_core::activity::ActivitySender; +use openshell_core::proto::ProviderProfileCredential; +use openshell_core::secrets::SecretResolver; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; + +type DynamicCredentials = Arc>>; + +enum PreparedHttpPolicy { + Inspect { + configs: Vec, + evaluator: Box, + }, + Passthrough { + generation_guard: PolicyGenerationGuard, + }, +} + +/// Everything an HTTP relay needs after authorization is complete. +/// +/// The relay deliberately owns a generation-pinned policy primitive instead +/// of retaining access to the mutable OPA engine. Policy reloads therefore +/// fail closed through the guard or tunnel evaluator already attached here. +pub(super) struct RelayContext<'a> { + request: &'a L7EvalContext, + policy: PreparedHttpPolicy, + middleware_engine: &'a OpaEngine, +} + +/// Build the request-processing context shared by CONNECT and forward HTTP. +pub(super) fn http_context( + decision: &EgressDecision, + secret_resolver: Option>, + activity_tx: Option, + dynamic_credentials: Option, +) -> L7EvalContext { + let policy_name = match &decision.action { + NetworkAction::Allow { matched_policy } => matched_policy.clone().unwrap_or_default(), + NetworkAction::Deny { .. } => String::new(), + }; + + L7EvalContext { + host: decision.intent.destination.host.clone(), + port: decision.intent.destination.port, + policy_name, + binary_path: decision + .binary + .as_ref() + .map(|path| path.to_string_lossy().into_owned()) + .unwrap_or_default(), + ancestors: decision + .ancestors + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(), + cmdline_paths: decision + .cmdline_paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(), + secret_resolver, + activity_tx, + dynamic_credentials: dynamic_credentials.clone(), + token_grant_resolver: dynamic_credentials + .as_ref() + .map(|_| crate::l7::token_grant_injection::default_resolver()), + } +} + +/// Pin a generation for a relay or the forward HTTP single-request path. +pub(super) fn pin_policy_generation( + opa_engine: &OpaEngine, + expected_generation: u64, +) -> Result { + opa_engine.generation_guard(expected_generation) +} + +/// Clone an L7 evaluator for a relay or the forward HTTP single-request path. +pub(super) fn pin_l7_evaluator( + opa_engine: &OpaEngine, + expected_generation: u64, +) -> Result { + opa_engine.clone_engine_for_tunnel(expected_generation) +} + +/// Prepare a generation-pinned HTTP relay at the adapter boundary. +/// +/// A stale generation preserves the established CONNECT behavior: emit the +/// policy-change close event and let the adapter close the live tunnel without +/// attempting to write an HTTP response into it. +pub(super) fn prepare_http_relay<'a>( + route: Option<&L7RouteSnapshot>, + opa_engine: &'a OpaEngine, + decision: &EgressDecision, + request: &'a L7EvalContext, +) -> Option> { + let policy = if let Some(route) = route.filter(|route| !route.configs.is_empty()) { + let evaluator = match pin_l7_evaluator(opa_engine, route.l7_policy_generation) { + Ok(evaluator) => evaluator, + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return None; + } + }; + let configs = route + .configs + .iter() + .map(|snapshot| snapshot.config.clone()) + .collect(); + PreparedHttpPolicy::Inspect { + configs, + evaluator: Box::new(evaluator), + } + } else { + let expected_generation = route.map_or(decision.l4_policy_generation, |route| { + route.l7_policy_generation + }); + let generation_guard = match pin_policy_generation(opa_engine, expected_generation) { + Ok(guard) => guard, + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return None; + } + }; + PreparedHttpPolicy::Passthrough { generation_guard } + }; + + Some(RelayContext { + request, + policy, + middleware_engine: opa_engine, + }) +} + +/// Pin the generation used by a raw relay so policy activation or quarantine +/// closes streams that otherwise have no request boundary at which to notice +/// a stale decision. +pub(super) fn prepare_raw_relay( + route: Option<&L7RouteSnapshot>, + opa_engine: &OpaEngine, + decision: &EgressDecision, +) -> Option { + let expected_generation = route.map_or(decision.l4_policy_generation, |route| { + route.l7_policy_generation + }); + match pin_policy_generation(opa_engine, expected_generation) { + Ok(guard) => Some(guard), + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + None + } + } +} + +/// Relay an HTTP/1 stream using an already-authorized, generation-pinned context. +/// +/// CONNECT plaintext and TLS-terminated streams both enter through this +/// function. Forward HTTP will provide a buffered first request to the same +/// boundary in the next migration step. +pub(super) async fn relay_http_stream( + client: &mut C, + upstream: &mut U, + context: RelayContext<'_>, +) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + match context.policy { + PreparedHttpPolicy::Inspect { configs, evaluator } if configs.len() == 1 => { + let generation_guard = evaluator.generation_guard().clone(); + tokio::select! { + result = crate::l7::relay::relay_with_inspection( + &configs[0], + *evaluator, + client, + upstream, + context.request, + ) => result, + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(context.request, &generation_guard); + Ok(()) + } + } + } + PreparedHttpPolicy::Inspect { configs, evaluator } => { + let generation_guard = evaluator.generation_guard().clone(); + tokio::select! { + result = crate::l7::relay::relay_with_route_selection( + &configs, + *evaluator, + client, + upstream, + context.request, + ) => result, + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(context.request, &generation_guard); + Ok(()) + } + } + } + PreparedHttpPolicy::Passthrough { generation_guard } => { + tokio::select! { + result = crate::l7::relay::relay_passthrough_with_credentials( + client, + upstream, + context.request, + &generation_guard, + Some(context.middleware_engine), + ) => result, + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(context.request, &generation_guard); + Ok(()) + } + } + } + } +} + +/// Relay a policy-authorized raw TCP stream. +pub(super) async fn relay_tcp( + client: &mut C, + upstream: &mut U, + generation_guard: &PolicyGenerationGuard, + request: &L7EvalContext, +) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin, + U: AsyncRead + AsyncWrite + Unpin, +{ + tokio::select! { + result = tokio::io::copy_bidirectional(client, upstream) => { + result.into_diagnostic()?; + } + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(request, generation_guard); + } + } + Ok(()) +} + +fn emit_stale_relay_close(request: &L7EvalContext, guard: &PolicyGenerationGuard) { + emit_l7_tunnel_close_after_policy_change( + &request.host, + request.port, + miette::miette!( + "policy generation is stale [captured_generation:{} current_generation:{}]", + guard.captured_generation(), + guard.current_generation(), + ), + ); +} + +#[cfg(test)] +mod tests { + use super::super::{EgressIntent, EndpointDecision, ProcessIdentityEvidence}; + use super::*; + + const POLICY_REGO: &str = include_str!("../../data/sandbox-policy.rego"); + const EMPTY_POLICY_DATA: &str = "network_policies: {}\n"; + + fn decision(l4_policy_generation: u64) -> EgressDecision { + EgressDecision { + intent: EgressIntent::connect("example.com".to_string(), 80), + action: NetworkAction::Allow { + matched_policy: Some("test".to_string()), + }, + l4_policy_generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + } + } + + fn request_context() -> L7EvalContext { + L7EvalContext { + host: "example.com".to_string(), + port: 80, + policy_name: "test".to_string(), + binary_path: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + } + } + + #[test] + fn relay_without_route_pins_l4_decision_generation() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(engine.current_generation()); + let request = request_context(); + + let context = prepare_http_relay(None, &engine, &decision, &request) + .expect("current L4 generation should prepare a relay"); + let PreparedHttpPolicy::Passthrough { generation_guard } = context.policy else { + panic!("route-less relay should use a generation guard"); + }; + + assert_eq!( + generation_guard.captured_generation(), + decision.l4_policy_generation + ); + } + + #[test] + fn empty_hydrated_route_pins_l7_lookup_generation() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(u64::MAX); + let route = L7RouteSnapshot { + configs: vec![], + l7_policy_generation: engine.current_generation(), + }; + let request = request_context(); + + let context = prepare_http_relay(Some(&route), &engine, &decision, &request) + .expect("the hydrated route generation should take precedence over stale L4 data"); + let PreparedHttpPolicy::Passthrough { generation_guard } = context.policy else { + panic!("empty L7 route should use a generation guard"); + }; + + assert_eq!( + generation_guard.captured_generation(), + route.l7_policy_generation + ); + } + + #[test] + fn stale_generation_fails_before_relay_context_is_created() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(engine.current_generation()); + let request = request_context(); + engine.reload(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + + assert!( + prepare_http_relay(None, &engine, &decision, &request).is_none(), + "policy reload must prevent a stale relay from starting" + ); + } + + #[tokio::test] + async fn raw_relay_closes_immediately_when_fail_closed_generation_is_published() { + let engine = Arc::new(OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap()); + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let request = request_context(); + let (_client_peer, mut proxy_client) = tokio::io::duplex(64); + let (_upstream_peer, mut proxy_upstream) = tokio::io::duplex(64); + + let relay = tokio::spawn(async move { + relay_tcp(&mut proxy_client, &mut proxy_upstream, &guard, &request).await + }); + tokio::task::yield_now().await; + + engine + .enter_fail_closed("candidate policy validation failed") + .unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("raw relay should close when its generation becomes stale") + .expect("relay task should not panic") + .expect("stale relay closure should be clean"); + } +} diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs new file mode 100644 index 0000000000..d4097b5938 --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -0,0 +1,655 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Compatibility and regression contracts for the shared proxy egress pipeline. + +use super::*; +use std::io::Write; +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing_subscriber::prelude::*; + +#[derive(Clone)] +struct SharedBuffer(Arc>>); + +impl Write for SharedBuffer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn allowed_decision(intent: EgressIntent) -> EgressDecision { + EgressDecision { + intent, + action: NetworkAction::Allow { + matched_policy: Some("proxy_compatibility".to_string()), + }, + l4_policy_generation: 0, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: Some(PathBuf::from("/usr/bin/curl")), + binary_pid: Some(42), + ancestors: vec![PathBuf::from("/usr/bin/sh")], + cmdline_paths: vec![], + } +} + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + (client, server) +} + +fn assert_json_response( + response: &[u8], + expected_status: &str, + expected_error: &str, + expected_detail: &str, +) { + let (headers, body) = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|end| (&response[..end + 4], &response[end + 4..])) + .expect("complete HTTP response"); + let headers = String::from_utf8(headers.to_vec()).unwrap(); + assert!(headers.starts_with(expected_status)); + assert!(headers.contains("Content-Type: application/json\r\n")); + assert!(headers.contains(&format!("Content-Length: {}\r\n", body.len()))); + assert!(headers.contains("Connection: close\r\n")); + assert_eq!( + serde_json::from_slice::(body).unwrap(), + serde_json::json!({ + "error": expected_error, + "detail": expected_detail, + }) + ); +} + +#[tokio::test] +async fn destination_denials_preserve_adapter_specific_wire_contracts() { + let cases = [ + ( + DestinationDenialKind::TrustedGateway, + "trusted-gateway check failed", + ), + ( + DestinationDenialKind::InvalidAllowedIps, + "invalid allowed_ips in policy", + ), + ( + DestinationDenialKind::AllowedIps, + "allowed_ips check failed", + ), + ( + DestinationDenialKind::DeclaredEndpoint, + "declared endpoint check failed", + ), + (DestinationDenialKind::InternalAddress, "internal address"), + ]; + + for (kind, detail) in cases { + let denial = DestinationDenial { + kind, + reason: "proxy compatibility destination failure".to_string(), + }; + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + + let (mut app, mut proxy) = tcp_pair().await; + deny_connect_destination( + &mut proxy, + &denial, + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl", + &allowed_decision(EgressIntent::connect("target.example".to_string(), 8443)), + &None, + &None, + ) + .await + .unwrap(); + proxy.shutdown().await.unwrap(); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_json_response( + &response, + "HTTP/1.1 403 Forbidden\r\n", + "ssrf_denied", + &format!("CONNECT target.example:8443 blocked: {detail}"), + ); + + let (mut app, mut proxy) = tcp_pair().await; + deny_forward_destination( + &mut proxy, + &denial, + peer, + "POST", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl", + "proxy_compatibility", + &allowed_decision(EgressIntent::forward_http( + "target.example".to_string(), + 8080, + )), + None, + None, + ) + .await + .unwrap(); + proxy.shutdown().await.unwrap(); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_json_response( + &response, + "HTTP/1.1 403 Forbidden\r\n", + "ssrf_denied", + &format!("POST target.example:8080 blocked: {detail}"), + ); + } +} + +#[test] +fn representative_adapter_denials_preserve_ocsf_fields() { + let captured = Arc::new(Mutex::new(Vec::new())); + let layer = openshell_ocsf::tracing_layers::OcsfJsonlLayer::new(SharedBuffer(captured.clone())); + let subscriber = tracing_subscriber::registry().with(layer); + let denial_reason = "target.example resolves to internal address 10.0.0.5"; + tracing::subscriber::with_default(subscriber, || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let denial = DestinationDenial { + kind: DestinationDenialKind::InternalAddress, + reason: denial_reason.to_string(), + }; + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + + let (_app, mut proxy) = tcp_pair().await; + deny_connect_destination( + &mut proxy, + &denial, + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + &allowed_decision(EgressIntent::connect("target.example".to_string(), 8443)), + &None, + &None, + ) + .await + .unwrap(); + + let (_app, mut proxy) = tcp_pair().await; + deny_forward_destination( + &mut proxy, + &denial, + peer, + "POST", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "proxy_compatibility", + &allowed_decision(EgressIntent::forward_http( + "target.example".to_string(), + 8080, + )), + None, + None, + ) + .await + .unwrap(); + }); + }); + + let bytes = captured.lock().unwrap().clone(); + let events: Vec = String::from_utf8(bytes) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(events.len(), 2); + + let connect = &events[0]; + assert_eq!(connect["class_name"], "Network Activity"); + assert_eq!(connect["activity_name"], "Open"); + assert_eq!(connect["action"], "Denied"); + assert_eq!(connect["disposition"], "Blocked"); + assert_eq!(connect["severity"], "Medium"); + assert_eq!(connect["status"], "Failure"); + assert_eq!(connect["dst_endpoint"]["domain"], "target.example"); + assert_eq!(connect["dst_endpoint"]["port"], 8443); + assert_eq!(connect["actor"]["process"]["name"], "/usr/bin/curl"); + assert_eq!(connect["actor"]["process"]["pid"], 42); + assert_eq!(connect["firewall_rule"]["name"], "-"); + assert_eq!(connect["firewall_rule"]["type"], "ssrf"); + assert_eq!( + connect["message"], + "CONNECT blocked: internal address target.example:8443" + ); + assert_eq!(connect["status_detail"], denial_reason); + + let forward = &events[1]; + assert_eq!(forward["class_name"], "HTTP Activity"); + assert_eq!(forward["activity_name"], "Other"); + assert_eq!(forward["action"], "Denied"); + assert_eq!(forward["disposition"], "Blocked"); + assert_eq!(forward["severity"], "Medium"); + assert_eq!(forward["status"], "Failure"); + assert_eq!(forward["dst_endpoint"]["domain"], "target.example"); + assert_eq!(forward["dst_endpoint"]["port"], 8080); + assert_eq!(forward["http_request"]["http_method"], "POST"); + assert_eq!(forward["firewall_rule"]["name"], "proxy_compatibility"); + assert_eq!(forward["firewall_rule"]["type"], "ssrf"); + assert_eq!( + forward["message"], + "FORWARD blocked: internal IP without allowed_ips for target.example:8080" + ); + assert_eq!(forward["status_detail"], denial_reason); +} + +#[test] +fn representative_adapter_allows_preserve_ocsf_fields() { + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + let connect = serde_json::to_value(build_connect_allow_ocsf_event( + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "proxy_compatibility", + true, + )) + .unwrap(); + assert_eq!(connect["class_name"], "Network Activity"); + assert_eq!(connect["activity_name"], "Open"); + assert_eq!(connect["action"], "Allowed"); + assert_eq!(connect["disposition"], "Allowed"); + assert_eq!(connect["severity"], "Informational"); + assert_eq!(connect["status"], "Success"); + assert_eq!(connect["dst_endpoint"]["domain"], "target.example"); + assert_eq!(connect["dst_endpoint"]["port"], 8443); + assert_eq!(connect["actor"]["process"]["name"], "/usr/bin/curl"); + assert_eq!(connect["firewall_rule"]["name"], "proxy_compatibility"); + assert_eq!(connect["firewall_rule"]["type"], "opa"); + assert_eq!(connect["message"], "CONNECT_L7 allowed target.example:8443"); + + let forward = serde_json::to_value(build_forward_allow_ocsf_event( + peer, + "GET", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "proxy_compatibility", + )) + .unwrap(); + assert_eq!(forward["class_name"], "HTTP Activity"); + assert_eq!(forward["activity_name"], "Other"); + assert_eq!(forward["action"], "Allowed"); + assert_eq!(forward["disposition"], "Allowed"); + assert_eq!(forward["severity"], "Informational"); + assert_eq!(forward["status"], "Success"); + assert_eq!(forward["dst_endpoint"]["domain"], "target.example"); + assert_eq!(forward["dst_endpoint"]["port"], 8080); + assert_eq!(forward["http_request"]["http_method"], "GET"); + assert_eq!(forward["firewall_rule"]["name"], "proxy_compatibility"); + assert_eq!(forward["firewall_rule"]["type"], "opa"); + assert_eq!( + forward["message"], + "FORWARD allowed GET target.example:8080/v1/items" + ); +} + +fn poisoned_engine() -> OpaEngine { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + r#" +network_policies: + proxy_compatibility: + name: proxy_compatibility + endpoints: + - host: target.example + port: 443 + protocol: rest + enforcement: enforce + tls: skip + allowed_ips: ["10.0.0.0/8"] + rules: + - allow: { method: GET, path: "/**" } + binaries: + - path: /usr/bin/curl +"#, + ) + .unwrap(); + engine.poison_lock_for_test(); + engine +} + +#[test] +fn l7_query_failure_preserves_l4_only_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert!(query_l7_route_snapshot(&engine, &decision, "target.example", 443).is_none()); +} + +#[test] +fn tls_query_failure_preserves_auto_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert_eq!( + query_tls_mode(&engine, &decision, "target.example", 443), + crate::l7::TlsMode::Auto + ); +} + +#[test] +fn allowed_ips_query_failure_preserves_empty_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert!(query_allowed_ips(&engine, &decision, "target.example", 443).is_empty()); +} + +#[test] +fn exact_host_query_failure_preserves_false_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert!(!query_exact_declared_endpoint_host( + &engine, + &decision, + "target.example", + 443 + )); +} + +#[test] +fn identity_required_policy_accepts_real_binary_and_rejects_empty_exec_path() { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + r#" +network_policies: + proxy_compatibility: + name: proxy_compatibility + endpoints: + - host: target.example + port: 443 + binaries: + - path: /usr/bin/curl +"#, + ) + .unwrap(); + let input = |binary_path: PathBuf| crate::opa::NetworkInput { + host: "target.example".to_string(), + port: 443, + binary_path, + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + assert!(matches!( + engine + .evaluate_network_action(&input(PathBuf::from("/usr/bin/curl"))) + .unwrap(), + NetworkAction::Allow { .. } + )); + assert!(matches!( + engine + .evaluate_network_action(&input(PathBuf::new())) + .unwrap(), + NetworkAction::Deny { .. } + )); +} + +#[cfg(not(target_os = "linux"))] +#[test] +fn identity_required_mode_is_explicitly_unsupported_off_linux() { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let decision = authorize_egress_intent( + "127.0.0.1:41000".parse().unwrap(), + &engine, + &BinaryIdentityCache::new(), + &AtomicU32::new(1), + EgressIntent::connect("target.example".to_string(), 443), + ); + + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + assert_eq!( + decision.identity, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::UnsupportedPlatform) + ); +} + +#[test] +fn forward_rewrite_does_not_treat_a_pipelined_request_as_body_overflow() { + let raw = b"GET http://target.example/allowed HTTP/1.1\r\n\ + Host: target.example\r\n\ + Connection: keep-alive\r\n\r\n\ + POST http://target.example/blocked HTTP/1.1\r\n\ + Host: target.example\r\n\ + Content-Length: 0\r\n\r\n"; + let rewritten = + rewrite_forward_request(raw, raw.len(), "/allowed", "target.example", None, false).unwrap(); + let rewritten = String::from_utf8(rewritten).unwrap(); + + assert!(rewritten.starts_with("GET /allowed HTTP/1.1\r\n")); + assert!(rewritten.contains("Connection: close\r\n")); + assert!(!rewritten.contains("POST http://target.example/blocked")); +} + +#[test] +fn forward_rewrite_trims_pipeline_after_content_length_body() { + let raw = b"POST http://target.example/allowed HTTP/1.1\r\n\ + Host: target.example\r\n\ + Content-Length: 4\r\n\r\n\ + body\ + GET http://target.example/blocked HTTP/1.1\r\n\ + Host: target.example\r\n\r\n"; + let rewritten = + rewrite_forward_request(raw, raw.len(), "/allowed", "target.example", None, false).unwrap(); + let rewritten = String::from_utf8(rewritten).unwrap(); + + assert!(rewritten.ends_with("\r\n\r\nbody")); + assert!(!rewritten.contains("GET http://target.example/blocked")); +} + +#[test] +fn forward_rewrite_trims_pipeline_after_complete_chunked_body() { + let raw = b"POST http://target.example/allowed HTTP/1.1\r\n\ + Host: target.example\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 4\r\nbody\r\n0\r\n\r\n\ + GET http://target.example/blocked HTTP/1.1\r\n\ + Host: target.example\r\n\r\n"; + let rewritten = + rewrite_forward_request(raw, raw.len(), "/allowed", "target.example", None, false).unwrap(); + let rewritten = String::from_utf8(rewritten).unwrap(); + + assert!(rewritten.ends_with("4\r\nbody\r\n0\r\n\r\n")); + assert!(!rewritten.contains("GET http://target.example/blocked")); +} + +#[tokio::test] +async fn forward_https_absolute_form_rejection_is_snapshotted() { + let (response, denial_stages) = drive_forward_through_handler( + " - { host: \"target.example\", port: 443 }\n", + "https://target.example/private", + ) + .await; + + assert_eq!( + response, + b"HTTP/1.1 400 Bad Request\r\nContent-Length: 27\r\n\r\nUse CONNECT for HTTPS URLs" + ); + assert!(denial_stages.is_empty()); +} + +async fn exercise_benchmark_request(proxy_addr: SocketAddr, target: SocketAddr, connect: bool) { + let mut client = TcpStream::connect(proxy_addr).await.unwrap(); + let authority = target.to_string(); + let request = if connect { + format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n") + } else { + format!( + "GET http://{authority}/proxy-baseline HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\n" + ) + }; + client.write_all(request.as_bytes()).await.unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 403 Forbidden")); +} + +/// Run with: +/// `cargo test -p openshell-supervisor-network proxy_performance_baseline -- --ignored --nocapture --test-threads=1` +#[test] +#[ignore = "manual proxy allocation/query/latency baseline"] +fn proxy_performance_baseline() { + temp_env::with_vars( + [( + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + Some("endpoint-only"), + )], + || { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() + .block_on(async { + // Benchmark the full fail-closed path using a declared loopback + // destination. This is deterministic and never opens a listener + // outside the local process, so it does not trigger host firewall + // prompts during manual baseline collection. + let target: SocketAddr = "127.0.0.1:18080".parse().unwrap(); + + let policy = format!( + r#" +network_policies: + proxy_compatibility: + name: proxy_compatibility + endpoints: + - host: {host} + port: {port} + tls: skip + binaries: + - path: "/**" +"#, + host = target.ip(), + port = target.port(), + ); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + include_str!("../../../data/sandbox-policy.rego"), + &policy, + false, + ) + .unwrap(), + ); + let proxy_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_addr = proxy_listener.local_addr().unwrap(); + let proxy_engine = engine.clone(); + let proxy_task = tokio::spawn(async move { + while let Ok((stream, _)) = proxy_listener.accept().await { + let engine = proxy_engine.clone(); + tokio::spawn(async move { + Box::pin(handle_tcp_connection( + stream, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + None, + None, + Arc::new(None), + None, + None, + None, + None, + )) + .await + .unwrap(); + }); + } + }); + + for connect in [true, false] { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + + let iterations = std::env::var("OPENSHELL_PROXY_BASELINE_ITERATIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(25); + let mut results = serde_json::Map::new(); + for (name, connect) in [("connect", true), ("forward", false)] { + crate::test_alloc::reset(); + crate::opa::reset_test_opa_query_count(); + let started = std::time::Instant::now(); + for _ in 0..iterations { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + let elapsed = started.elapsed(); + let queries = crate::opa::test_opa_query_count(); + let (allocations, allocated_bytes) = crate::test_alloc::snapshot(); + let expected_queries = 4; + assert_eq!(queries, expected_queries * iterations); + results.insert( + name.to_string(), + serde_json::json!({ + "allocated_bytes_per_request": allocated_bytes / iterations, + "allocations_per_request": allocations / iterations, + "latency_ns_per_request": elapsed.as_nanos() / u128::from(iterations), + "opa_queries_per_request": queries / iterations, + }), + ); + } + println!( + "{}", + serde_json::json!({ + "iterations": iterations, + "proxy_performance_baseline": results, + "scenario": "declared_loopback_destination_denied", + "schema_version": 1, + }) + ); + + proxy_task.abort(); + }); + }, + ); +} diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs index 7d09d36f09..3dda53b2c3 100644 --- a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs +++ b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs @@ -77,6 +77,62 @@ pub fn parse_kmsg_line(line: &str, namespace_prefix: &str) -> Option (openshell_ocsf::OcsfEvent, openshell_ocsf::OcsfEvent) { + let hint = hint_for_event(event); + let reason = "direct connection bypassed HTTP CONNECT proxy"; + let dst_port = event.dst_port.to_string(); + let dst_ep = if let Ok(ip) = event.dst_addr.parse::() { + Endpoint::from_ip(ip, event.dst_port) + } else { + Endpoint::from_domain(&event.dst_addr, event.dst_port) + }; + + let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Refuse) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .dst_endpoint(dst_ep) + .actor_process(Process::from_bypass(binary, binary_pid, ancestors)) + .firewall_rule("bypass-detect", "nftables") + .observation_point(3) + .message(format!( + "BYPASS_DETECT {}:{} proto={} binary={binary} action=reject reason={reason}", + event.dst_addr, event.dst_port, event.proto, + )) + .build(); + + let finding_event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .is_alert(true) + .confidence(ConfidenceId::High) + .finding_info(FindingInfo::new("bypass-detect", "Proxy Bypass Detected").with_desc(reason)) + .remediation(hint) + .evidence_pairs(&[ + ("dst_addr", event.dst_addr.as_str()), + ("dst_port", dst_port.as_str()), + ("proto", event.proto.as_str()), + ("binary", binary), + ("binary_pid", binary_pid), + ("ancestors", ancestors), + ]) + .message(format!( + "BYPASS_DETECT {}:{} proto={} binary={binary} hint={hint}", + event.dst_addr, event.dst_port, event.proto, + )) + .build(); + + (net_event, finding_event) +} + /// Extract a single space-delimited field value from a nftables log line. /// /// Given `"DST="` and a string like `"...DST=93.184.216.34 LEN=60..."`, @@ -207,60 +263,11 @@ pub fn spawn( ("-".to_string(), "-".to_string(), "-".to_string()) }; - let hint = hint_for_event(&event); - let reason = "direct connection bypassed HTTP CONNECT proxy"; - // Dual-emit: Network Activity [4001] + Detection Finding [2004] - { - let dst_ep = if let Ok(ip) = event.dst_addr.parse::() { - Endpoint::from_ip(ip, event.dst_port) - } else { - Endpoint::from_domain(&event.dst_addr, event.dst_port) - }; - - let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Refuse) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .dst_endpoint(dst_ep.clone()) - .actor_process(Process::from_bypass(&binary, &binary_pid, &ancestors)) - .firewall_rule("bypass-detect", "nftables") - .observation_point(3) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} action=reject reason={reason}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - ocsf_emit!(net_event); - - let finding_event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .is_alert(true) - .confidence(ConfidenceId::High) - .finding_info( - FindingInfo::new("bypass-detect", "Proxy Bypass Detected") - .with_desc(reason), - ) - .remediation(hint) - .evidence_pairs(&[ - ("dst_addr", &event.dst_addr), - ("dst_port", &event.dst_port.to_string()), - ("proto", &event.proto), - ("binary", &binary), - ("binary_pid", &binary_pid), - ("ancestors", &ancestors), - ]) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} hint={hint}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - ocsf_emit!(finding_event); - } + let (net_event, finding_event) = + build_bypass_ocsf_events(&event, &binary, &binary_pid, &ancestors); + ocsf_emit!(net_event); + ocsf_emit!(finding_event); // Send to denial aggregator if available. if let Some(ref tx) = denial_tx { @@ -488,6 +495,49 @@ mod tests { assert!(hint_for_event(&event).contains("UDP")); } + #[test] + fn bypass_ocsf_contract_is_stable() { + let event = BypassEvent { + dst_addr: "93.184.216.34".to_string(), + dst_port: 443, + src_port: 48012, + proto: "tcp".to_string(), + uid: Some(1000), + }; + let (network, finding) = + build_bypass_ocsf_events(&event, "/usr/bin/curl", "42", "/usr/bin/sh"); + let network = serde_json::to_value(network).unwrap(); + assert_eq!(network["class_name"], "Network Activity"); + assert_eq!(network["activity_name"], "Refuse"); + assert_eq!(network["action"], "Denied"); + assert_eq!(network["disposition"], "Blocked"); + assert_eq!(network["severity"], "Medium"); + assert!(network.get("status").is_none()); + assert_eq!(network["dst_endpoint"]["ip"], "93.184.216.34"); + assert_eq!(network["dst_endpoint"]["port"], 443); + assert_eq!(network["actor"]["process"]["name"], "/usr/bin/curl"); + assert_eq!(network["firewall_rule"]["name"], "bypass-detect"); + assert_eq!(network["firewall_rule"]["type"], "nftables"); + assert_eq!(network["observation_point_id"], 3); + assert!( + network["message"] + .as_str() + .unwrap() + .contains("action=reject") + ); + + let finding = serde_json::to_value(finding).unwrap(); + assert_eq!(finding["class_name"], "Detection Finding"); + assert_eq!(finding["action"], "Denied"); + assert_eq!(finding["disposition"], "Blocked"); + assert_eq!(finding["severity"], "Medium"); + assert_eq!(finding["confidence"], "High"); + assert_eq!(finding["is_alert"], true); + assert_eq!(finding["finding_info"]["uid"], "bypass-detect"); + assert_eq!(finding["finding_info"]["title"], "Proxy Bypass Detected"); + assert_eq!(finding["evidences"][0]["data"]["dst_port"], "443"); + } + #[test] fn resolve_process_identity_surfaces_ambiguous_shared_socket() { use std::ffi::CString; diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 8723535d1a..1bb1514f1b 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -214,6 +214,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.oidc.rolesClaim | string | `""` | Dot-separated path to the roles array in the JWT claims. Keycloak: "realm_access.roles", Entra ID: "roles", Okta: "groups". | | server.oidc.scopesClaim | string | `""` | Dot-separated path to the scopes array in the JWT claims. | | server.oidc.userRole | string | `""` | Role name for standard user access. | +| server.policyValidationFailureMode | string | `"fail_closed"` | Posture when a candidate sandbox policy fails validation. `fail_closed` deactivates the previous policy; `retain_last_valid` keeps it active. | | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 36a579250b..ad84722677 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -33,6 +33,11 @@ data: {{- end }} log_level = {{ .Values.server.logLevel | quote }} sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} + {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} + {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} + {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} + {{- end }} + policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} default_image = {{ .Values.server.sandboxImage | quote }} {{- if include "openshell.supervisorImageOverrideEnabled" . }} supervisor_image = {{ include "openshell.supervisorImage" . | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index aee396c38f..90e4f9cef0 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -229,6 +229,22 @@ tests: path: data["gateway.toml"] pattern: 'grpc_rate_limit_window_seconds\s*=' + - it: renders fail-closed policy validation posture by default + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\].*?policy_validation_failure_mode\s*=\s*"fail_closed"' + + - it: renders retain-last-valid policy validation posture + template: templates/gateway-config.yaml + set: + server.policyValidationFailureMode: retain_last_valid + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\].*?policy_validation_failure_mode\s*=\s*"retain_last_valid"' + - it: renders the gRPC rate limit under [openshell.gateway] when both values are positive template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index e89a234912..351f4841eb 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -224,6 +224,9 @@ server: # -- Enable plaintext HTTP routing for loopback sandbox service URLs on # TLS-enabled gateways. enableLoopbackServiceHttp: true + # -- Posture when a candidate sandbox policy fails validation. `fail_closed` + # deactivates the previous policy; `retain_last_valid` keeps it active. + policyValidationFailureMode: fail_closed # Optional gateway-wide gRPC request rate limit. Applies only to gRPC API # traffic after protocol multiplexing; health, metrics, and loopback service # HTTP routes are not rate limited. Both values must be positive to enable the diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 3dde7643d5..7468a4f153 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -75,6 +75,10 @@ compute_drivers = ["kubernetes"] sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 +# Reject invalid policy generations securely by default. Set +# "retain_last_valid" only when availability takes priority. +policy_validation_failure_mode = "fail_closed" + # Subject Alternative Names baked into the gateway server certificate. # Wildcard DNS SANs (e.g. "*.dev.openshell.localhost") also enable sandbox # service URLs under that domain. @@ -171,6 +175,8 @@ phases = ["validate"] Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. +`[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. + `[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 990f260cea..428ec5673d 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -61,8 +61,7 @@ network_middlewares: Static sections are locked at sandbox creation. Changing them requires destroying and recreating the sandbox. Dynamic sections can be updated on a running sandbox with `openshell policy update` for incremental merges or `openshell policy set` for full replacement, and take effect without restarting. -When a hot reload changes rules on an active HTTP L7 endpoint, existing keep-alive tunnels are closed before forwarding another parsed request. Credential-injection-only HTTP passthrough tunnels use the same reload boundary. Most HTTP clients reconnect automatically, and the next request is evaluated against the current policy. -Raw streams are connection-scoped and outside L7 live-reload guarantees. This includes `tls: skip`, non-HTTP TCP payloads, HTTP upgrades such as WebSocket, and long-lived response streams such as SSE. A reload applies to the next connection or next parsed HTTP request; it does not interrupt an already-forwarded raw stream. Use `protocol: websocket` when policy should stay attached to the RFC 6455 upgrade and client text messages after the allowed upgrade. Add `websocket_credential_rewrite: true` only when the relay should rewrite credential placeholders in client-to-server WebSocket text messages. Add `request_body_credential_rewrite: true` only on inspected REST endpoints that need OpenShell to rewrite placeholders in supported text request bodies. +When a hot reload changes rules, the supervisor publishes a new policy generation and closes connections pinned to the previous generation. This includes HTTP keep-alive tunnels, `tls: skip`, non-HTTP payloads, HTTP upgrades such as WebSocket, and long-lived response streams such as SSE. Most clients reconnect automatically, and the next connection or request is evaluated against the current policy. Use `protocol: websocket` when policy should stay attached to the RFC 6455 upgrade and client text messages after the allowed upgrade. Add `websocket_credential_rewrite: true` only when the relay should rewrite credential placeholders in client-to-server WebSocket text messages. Add `request_body_credential_rewrite: true` only on inspected REST endpoints that need OpenShell to rewrite placeholders in supported text request bodies. | Section | Type | Description | |---|---|---| @@ -202,6 +201,30 @@ The following steps outline the hot-reload policy update workflow. openshell policy list ``` +### Validation failures + +OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. OpenShell rejects the candidate when overlapping exact or wildcard host selectors disagree on those fields. Path-scoped endpoints may use different protocols only when their path selectors cannot match the same request. + +The gateway's `policy_validation_failure_mode` configuration determines what happens after rejection. Set it under `[openshell.gateway]` in `gateway.toml`. Its default is `fail_closed`: + +```toml +[openshell.gateway] +policy_validation_failure_mode = "fail_closed" +``` + +In `fail_closed` mode, the supervisor publishes a quarantine generation, denies new egress, and closes connections pinned to the previous generation. The previous policy is not active. A later valid policy exits quarantine automatically. + +Operators that explicitly prioritize availability can retain the previous generation: + +```toml +[openshell.gateway] +policy_validation_failure_mode = "retain_last_valid" +``` + +In `retain_last_valid` mode, the rejected candidate remains inactive and the previous valid generation remains active. If no previous valid generation exists, such as during initial startup, OpenShell still fails closed. Restart the gateway after changing `gateway.toml`; connected sandbox supervisors receive the configured posture from the restarted gateway. Individual sandboxes cannot override it. + +OCSF configuration and finding events identify the rejected candidate, validation rationale, configured and effective modes, active generation, and whether the previous policy is active. When `retain_last_valid` is configured without a previous valid generation, the effective mode remains `fail_closed`. Connection denials during quarantine include the validation failure as their policy denial rationale. + ## Incremental Policy Updates Use `openshell policy update` when you want to merge network policy changes into the current live policy instead of replacing the whole YAML document. This command only updates the dynamic `network_policies` section. diff --git a/e2e/python/test_sandbox_policy.py b/e2e/python/test_sandbox_policy.py index 5ac37bd27f..82e32a9b34 100644 --- a/e2e/python/test_sandbox_policy.py +++ b/e2e/python/test_sandbox_policy.py @@ -1950,15 +1950,14 @@ def test_host_wildcard_rejects_deep_subdomain( # ============================================================================= -def test_overlapping_policies_do_not_crash_opa( +def test_overlapping_policies_with_conflicting_destination_metadata_are_rejected( sandbox: Callable[..., Sandbox], ) -> None: - """OVL-1: Two policies covering the same host:port must not crash OPA. + """OVL-1: Conflicting metadata on the same host:port fails closed. - After a draft rule approval, the merged policy can contain two entries - for the same (host, port). The OPA engine must handle this without - a 'duplicated definition of local variable' error. This test creates - the overlap directly to simulate the post-approval state. + One endpoint permits any resolved address while the other constrains + ``allowed_ips``. The complete candidate is ambiguous and must not activate + either entry. """ policy = _base_policy( network_policies={ @@ -1992,8 +1991,9 @@ def test_overlapping_policies_do_not_crash_opa( args=(_PROXY_HOST, _PROXY_PORT, _SANDBOX_IP, _FORWARD_PROXY_PORT), ) assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, ( - f"Overlapping policies should not crash; expected 200, got: {result.stdout}" + assert "403" in result.stdout, ( + "Conflicting overlapping policies should fail closed; " + f"expected 403, got: {result.stdout}" ) diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 6e8fa70bdf..ec626eb78e 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -102,6 +102,11 @@ name = "forward_proxy_jsonrpc_l7" path = "tests/forward_proxy_jsonrpc_l7.rs" required-features = ["e2e-host-gateway"] +[[test]] +name = "proxy_egress_pipeline" +path = "tests/proxy_egress_pipeline.rs" +required-features = ["e2e-host-gateway"] + [[test]] name = "gpu" path = "tests/gpu.rs" diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs new file mode 100644 index 0000000000..28d2d6f1d5 --- /dev/null +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -0,0 +1,1758 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e")] + +//! E2E coverage for the shared explicit-proxy egress pipeline. +//! +//! These tests exercise behavior that must remain identical while CONNECT and +//! forward HTTP converge on shared authorization, destination, and relay +//! primitives: +//! - live policy reloads affect new requests through both adapters and close a +//! pre-existing CONNECT HTTP stream before its next request is forwarded; +//! - `tls: skip` selects a byte-transparent TCP relay; +//! - provider placeholders in HTTP headers and opted-in REST bodies are +//! resolved through both adapters without appearing in test output. + +use std::io::{self, Error, ErrorKind, Write}; +use std::process::Stdio; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serde_json::Value; +use tempfile::NamedTempFile; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; + +const TEST_SERVER_HOST: &str = "host.openshell.internal"; +const PROVIDER_NAME: &str = "e2e-proxy-egress-credentials"; +const TOKEN_ENV: &str = "PROXY_E2E_TOKEN"; +const TEST_SECRET: &str = "sk-e2e-proxy-egress-secret"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; +const PRIVATE_ALLOWED_IPS: &str = r#" allowed_ips: + - "10.0.0.0/8" + - "172.0.0.0/8" + - "192.168.0.0/16" + - "fc00::/7""#; +static PROVIDER_LOCK: Mutex<()> = Mutex::new(()); + +async fn run_cli(args: &[&str]) -> Result { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + + let output = cmd + .output() + .await + .map_err(|error| format!("failed to spawn openshell {}: {error}", args.join(" ")))?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}{stderr}"); + + if !output.status.success() { + return Err(format!( + "openshell {} failed (exit {:?}):\n{combined}", + args.join(" "), + output.status.code() + )); + } + + Ok(combined) +} + +async fn delete_provider(name: &str) { + let mut cmd = openshell_cmd(); + cmd.args(["provider", "delete", name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let _ = cmd.status().await; +} + +async fn create_generic_provider(name: &str) -> Result { + let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); + run_cli(&[ + "provider", + "create", + "--name", + name, + "--type", + "generic", + "--credential", + &credential, + ]) + .await +} + +fn write_policy_document( + host: &str, + port: u16, + endpoint_options: &str, + network_middlewares: &str, +) -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +{network_middlewares} +network_policies: + proxy_egress_test: + name: proxy_egress_test + endpoints: + - host: {host} + port: {port} +{endpoint_options} +{PRIVATE_ALLOWED_IPS} + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn write_policy(host: &str, port: u16, endpoint_options: &str) -> Result { + write_policy_document(host, port, endpoint_options, "") +} + +fn write_middleware_policy( + host: &str, + port: u16, + endpoint_options: &str, + on_error: &str, +) -> Result { + let network_middlewares = format!( + r#"network_middlewares: + regex-redactor: + name: Redact API tokens + middleware: openshell/regex + order: 10 + config: + mode: redact + on_error: {on_error} + endpoints: + include: ["{host}"] + exclude: [] +"# + ); + write_policy_document(host, port, endpoint_options, &network_middlewares) +} + +fn write_denied_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: {} +"#; + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn write_ambiguous_policy(host: &str, port: u16) -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + terminating: + name: terminating + endpoints: + - host: {host} + port: {port} +{PRIVATE_ALLOWED_IPS} + binaries: + - path: "/**" + passthrough: + name: passthrough + endpoints: + - host: {host} + port: {port} + tls: skip +{PRIVATE_ALLOWED_IPS} + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn write_destination_denial_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + destination_denials: + name: destination_denials + endpoints: + - { host: 169.254.169.254, port: 80 } + - { host: 127.0.0.1, port: 80 } + - { host: 203.0.113.10, port: 6443 } + - host: 203.0.113.10 + port: 8080 + allowed_ips: ["198.51.100.0/24"] + binaries: + - path: "/**" +"#; + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn write_ip_literal_success_policy( + ip: &str, + explicit_port: u16, + implicit_port: u16, +) -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + destination_successes: + name: destination_successes + endpoints: + - host: {ip} + port: {explicit_port} + allowed_ips: ["{ip}/32"] + - host: {ip} + port: {implicit_port} + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn policy_path(file: &NamedTempFile) -> String { + file.path() + .to_str() + .expect("temporary policy path should be utf-8") + .to_string() +} + +async fn read_until(stream: &mut TcpStream, marker: &[u8]) -> io::Result> { + let mut data = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let read = stream.read(&mut buffer).await?; + if read == 0 { + return Ok(data); + } + data.extend_from_slice(&buffer[..read]); + if data.windows(marker.len()).any(|window| window == marker) { + return Ok(data); + } + } +} + +fn header_end(bytes: &[u8]) -> Option { + bytes + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) +} + +fn content_length(headers: &[u8]) -> io::Result { + let text = + std::str::from_utf8(headers).map_err(|error| Error::new(ErrorKind::InvalidData, error))?; + Ok(text + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.trim() + .eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0)) +} + +async fn read_http_request(stream: &mut TcpStream) -> io::Result>> { + let mut request = read_until(stream, b"\r\n\r\n").await?; + if request.is_empty() { + return Ok(None); + } + let headers_end = header_end(&request) + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; + let body_length = content_length(&request[..headers_end])?; + let total_length = headers_end + body_length; + while request.len() < total_length { + let mut buffer = vec![0_u8; total_length - request.len()]; + let read = stream.read(&mut buffer).await?; + if read == 0 { + return Err(Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP body")); + } + request.extend_from_slice(&buffer[..read]); + } + request.truncate(total_length); + Ok(Some(request)) +} + +struct KeepAliveHttpServer { + port: u16, + connections: Arc, + task: JoinHandle<()>, +} + +impl KeepAliveHttpServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind HTTP server: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read HTTP server address: {error}"))? + .port(); + let connections = Arc::new(AtomicUsize::new(0)); + let task_connections = Arc::clone(&connections); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + task_connections.fetch_add(1, Ordering::AcqRel); + tokio::spawn(async move { + let _ = handle_keep_alive_connection(stream).await; + }); + } + }); + Ok(Self { + port, + connections, + task, + }) + } + + fn connection_count(&self) -> usize { + self.connections.load(Ordering::Acquire) + } +} + +impl Drop for KeepAliveHttpServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn handle_keep_alive_connection(mut stream: TcpStream) -> io::Result<()> { + while let Some(request) = read_http_request(&mut stream).await? { + let close = String::from_utf8_lossy(&request) + .lines() + .any(|line| line.eq_ignore_ascii_case("connection: close")); + let connection = if close { "close" } else { "keep-alive" }; + let response = + format!("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\nok"); + stream.write_all(response.as_bytes()).await?; + if close { + return Ok(()); + } + } + Ok(()) +} + +struct EchoServer { + port: u16, + observed: Arc>>, + task: JoinHandle<()>, +} + +struct RequestBodyEchoServer { + port: u16, + task: JoinHandle<()>, +} + +impl RequestBodyEchoServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind request body echo server: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read request body echo server address: {error}"))? + .port(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + let _ = handle_request_body_echo(stream).await; + }); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for RequestBodyEchoServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn handle_request_body_echo(mut stream: TcpStream) -> io::Result<()> { + let request = read_http_request(&mut stream) + .await? + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; + let headers_end = header_end(&request) + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; + let body = &request[headers_end..]; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(response.as_bytes()).await?; + stream.write_all(body).await +} + +struct PipelineProbeServer { + port: u16, + observed: Arc>>, + task: JoinHandle<()>, +} + +impl PipelineProbeServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind pipeline probe: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read pipeline probe address: {error}"))? + .port(); + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_task = observed.clone(); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let observed = observed_task.clone(); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + match tokio::time::timeout( + std::time::Duration::from_millis(200), + stream.read(&mut buffer), + ) + .await + { + Ok(Ok(0)) | Err(_) => break, + Ok(Ok(read)) => request.extend_from_slice(&buffer[..read]), + Ok(Err(_)) => return, + } + } + observed.lock().unwrap().extend_from_slice(&request); + let _ = stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .await; + }); + } + }); + Ok(Self { + port, + observed, + task, + }) + } + + fn observed_request(&self) -> Vec { + self.observed.lock().unwrap().clone() + } +} + +impl Drop for PipelineProbeServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl EchoServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind echo server: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read echo server address: {error}"))? + .port(); + let observed = Arc::new(Mutex::new(Vec::new())); + let task_observed = Arc::clone(&observed); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let observed = Arc::clone(&task_observed); + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + let Ok(read) = stream.read(&mut buffer).await else { + break; + }; + if read == 0 { + break; + } + observed.lock().unwrap().extend_from_slice(&buffer[..read]); + if stream.write_all(&buffer[..read]).await.is_err() { + break; + } + } + }); + } + }); + Ok(Self { + port, + observed, + task, + }) + } + + fn observed_bytes(&self) -> Vec { + self.observed.lock().unwrap().clone() + } +} + +impl Drop for EchoServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +struct CredentialProbeServer { + port: u16, + task: JoinHandle<()>, +} + +impl CredentialProbeServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind credential probe: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read credential probe address: {error}"))? + .port(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + let _ = handle_credential_probe(stream).await; + }); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for CredentialProbeServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn handle_credential_probe(mut stream: TcpStream) -> io::Result<()> { + let request = read_http_request(&mut stream) + .await? + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; + let headers_end = header_end(&request) + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; + let headers = String::from_utf8_lossy(&request[..headers_end]); + let expected_authorization = format!("Bearer {TEST_SECRET}"); + let header_resolved = headers.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("authorization") && value.trim() == expected_authorization + }) + }); + let body_resolved = request[headers_end..] + .windows(TEST_SECRET.len()) + .any(|window| window == TEST_SECRET.as_bytes()); + let saw_placeholder = request + .windows(PLACEHOLDER_PREFIX.len()) + .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()); + let body = serde_json::json!({ + "body_resolved": body_resolved, + "header_resolved": header_resolved, + "saw_placeholder": saw_placeholder, + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await +} + +fn proxy_status_script(host: &str, port: u16) -> String { + format!( + r#" +import json +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} + +def proxy_parts(): + proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) + ) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_headers(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def status(response): + parts = response.split(None, 2) + return int(parts[1]) if len(parts) > 1 else 0 + +def forward_status(): + proxy_host, proxy_port = proxy_parts() + target = f"{{HOST}}:{{PORT}}" + with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: + sock.sendall( + f"GET http://{{target}}/forward HTTP/1.1\r\n" + f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + +def connect_status(): + proxy_host, proxy_port = proxy_parts() + target = f"{{HOST}}:{{PORT}}" + with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + code = status(read_headers(sock)) + if code != 200: + return code + sock.sendall( + f"GET /connect HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + +print(json.dumps({{"connect": connect_status(), "forward": forward_status()}}, sort_keys=True)) +"#, + host = host, + port = port, + ) +} + +fn persistent_connect_script(host: &str, port: u16) -> String { + format!( + r#" +import json +import os +import socket +import time +import urllib.parse + +HOST = {host:?} +PORT = {port} +READY = "/tmp/proxy-reload-ready" +GO = "/tmp/proxy-reload-go" +RESULT = "/tmp/proxy-reload-result" + +def proxy_parts(): + proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) + ) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + return 0 + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + return 0 + body += chunk + return int(headers.split(None, 2)[1]) + +proxy_host, proxy_port = proxy_parts() +target = f"{{HOST}}:{{PORT}}" +failed_closed = False +second_status = 0 +try: + with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + if read_response(sock) != 200: + raise RuntimeError("initial CONNECT was denied") + sock.sendall( + f"GET /before-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: keep-alive\r\n\r\n".encode() + ) + if read_response(sock) != 200: + raise RuntimeError("initial tunneled request was denied") + open(READY, "w").close() + deadline = time.monotonic() + 120 + while not os.path.exists(GO) and time.monotonic() < deadline: + time.sleep(0.1) + if not os.path.exists(GO): + raise RuntimeError("timed out waiting for policy reload signal") + try: + sock.sendall( + f"GET /after-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + second_status = read_response(sock) + except OSError: + second_status = 0 + failed_closed = second_status != 200 +finally: + with open(RESULT, "w") as result: + json.dump({{"failed_closed": failed_closed, "second_status": second_status}}, result, sort_keys=True) +"#, + host = host, + port = port, + ) +} + +async fn wait_for_sandbox_file(guard: &SandboxGuard, path: &str, log_path: &str) -> String { + let script = format!( + r#"import os, time +deadline = time.monotonic() + 60 +while not os.path.exists({path:?}) and time.monotonic() < deadline: + time.sleep(0.1) +if not os.path.exists({path:?}): + if os.path.exists({log_path:?}): + print(open({log_path:?}).read()) + raise SystemExit("timed out waiting for {path}") +print(open({path:?}).read()) +"# + ); + guard + .exec(&["python3", "-c", &script]) + .await + .unwrap_or_else(|error| panic!("wait for sandbox file {path}: {error}")) +} + +fn parse_json_line(output: &str) -> Value { + output + .lines() + .filter_map(|line| serde_json::from_str::(line.trim()).ok()) + .next_back() + .unwrap_or_else(|| panic!("missing JSON result in sandbox output:\n{output}")) +} + +#[tokio::test] +async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { + let server = KeepAliveHttpServer::start() + .await + .expect("start keep-alive HTTP server"); + let policy_a = write_policy(TEST_SERVER_HOST, server.port, "").expect("write policy A"); + let policy_b = write_denied_policy().expect("write policy B"); + let policy_a_path = policy_path(&policy_a); + let policy_b_path = policy_path(&policy_b); + + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &policy_a_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &policy_a_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("wait for policy A"); + + let persistent_script = persistent_connect_script(TEST_SERVER_HOST, server.port); + guard + .exec(&[ + "sh", + "-c", + "nohup python3 -c \"$1\" >/tmp/proxy-reload-client.log 2>&1 &", + "proxy-reload-client", + &persistent_script, + ]) + .await + .expect("start persistent CONNECT client"); + wait_for_sandbox_file( + &guard, + "/tmp/proxy-reload-ready", + "/tmp/proxy-reload-client.log", + ) + .await; + + let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let before = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters before reload"); + let before = parse_json_line(&before); + assert_eq!(before["connect"], 200, "CONNECT before reload: {before}"); + assert_eq!( + before["forward"], 200, + "forward HTTP before reload: {before}" + ); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &policy_b_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("publish and wait for policy B"); + + guard + .exec(&["sh", "-c", "touch /tmp/proxy-reload-go"]) + .await + .expect("release persistent CONNECT client"); + let stale_tunnel = wait_for_sandbox_file( + &guard, + "/tmp/proxy-reload-result", + "/tmp/proxy-reload-client.log", + ) + .await; + let stale_tunnel = parse_json_line(&stale_tunnel); + assert_eq!( + stale_tunnel["failed_closed"], true, + "existing CONNECT HTTP stream forwarded after policy reload: {stale_tunnel}" + ); + + let after = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters after reload"); + let after = parse_json_line(&after); + assert_eq!(after["connect"], 403, "CONNECT after reload: {after}"); + assert_eq!(after["forward"], 403, "forward HTTP after reload: {after}"); + + guard.cleanup().await; +} + +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn ambiguous_policy_update_fails_closed_without_contacting_upstream() { + let server = KeepAliveHttpServer::start() + .await + .expect("start keep-alive HTTP server"); + let valid_policy = write_policy(TEST_SERVER_HOST, server.port, "").expect("write valid policy"); + let ambiguous_policy = + write_ambiguous_policy(TEST_SERVER_HOST, server.port).expect("write ambiguous policy"); + let valid_policy_path = policy_path(&valid_policy); + let ambiguous_policy_path = policy_path(&ambiguous_policy); + + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &valid_policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &valid_policy_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("wait for valid policy"); + + let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let before = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters before invalid update"); + let before = parse_json_line(&before); + assert_eq!(before["connect"], 200, "CONNECT before update: {before}"); + assert_eq!(before["forward"], 200, "forward before update: {before}"); + let connections_before_rejection = server.connection_count(); + + let update_error = run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &ambiguous_policy_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect_err("ambiguous policy must report a failed revision"); + assert!( + update_error.contains("ambiguity validation failed"), + "policy update should explain the ambiguity:\n{update_error}" + ); + + let quarantined = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters in fail-closed quarantine"); + let quarantined = parse_json_line(&quarantined); + assert_eq!( + quarantined["connect"], 403, + "CONNECT in quarantine: {quarantined}" + ); + assert_eq!( + quarantined["forward"], 403, + "forward in quarantine: {quarantined}" + ); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert_eq!( + server.connection_count(), + connections_before_rejection, + "quarantined requests must not contact the upstream server" + ); + + let logs = run_cli(&[ + "logs", + &guard.name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await + .expect("fetch sandbox logs after rejection"); + assert!( + logs.contains("configured_mode=fail_closed effective_mode=fail_closed"), + "OCSF logs should state the effective validation posture:\n{logs}" + ); + assert!( + logs.contains("previous policy IS NOT active"), + "OCSF logs should state that the previous policy is inactive:\n{logs}" + ); + assert!( + logs.contains("conflicting metadata") && logs.contains("tls"), + "OCSF logs should contain the overlap rationale:\n{logs}" + ); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &valid_policy_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("valid policy should exit quarantine"); + let recovered = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters after recovery"); + let recovered = parse_json_line(&recovered); + assert_eq!( + recovered["connect"], 200, + "CONNECT after recovery: {recovered}" + ); + assert_eq!( + recovered["forward"], 200, + "forward after recovery: {recovered}" + ); + + guard.cleanup().await; +} + +#[tokio::test] +async fn destination_denial_modes_match_across_connect_and_forward_adapters() { + let policy = write_destination_denial_policy().expect("write destination denial policy"); + let policy_path = policy_path(&policy); + let script = r#" +import json +import os +import socket +import urllib.parse + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + headers, _, body = data.partition(b"\r\n\r\n") + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + status = int(headers.split(None, 2)[1]) + return {"status": status, "body": json.loads(body.decode())} + +def connect_result(host, port): + with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{host}:{port}" + sock.sendall(f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n".encode()) + return read_response(sock) + +def forward_result(host, port): + with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{host}:{port}" + sock.sendall( + f"GET http://{target}/probe HTTP/1.1\r\n" + f"Host: {target}\r\nConnection: close\r\n\r\n".encode() + ) + return read_response(sock) + +targets = { + "metadata": ("169.254.169.254", 80), + "loopback": ("127.0.0.1", 80), + "control_plane": ("203.0.113.10", 6443), + "outside_allowed_ips": ("203.0.113.10", 8080), +} +result = {} +for name, target in targets.items(): + result[name] = { + "connect": connect_result(*target), + "forward": forward_result(*target), + } +print(json.dumps(result, sort_keys=True)) +"#; + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", script]) + .await + .expect("sandbox create"); + let result = parse_json_line(&guard.create_output); + for name in [ + "metadata", + "loopback", + "control_plane", + "outside_allowed_ips", + ] { + for adapter in ["connect", "forward"] { + assert_eq!( + result[name][adapter]["status"], 403, + "{name} {adapter}: {result}" + ); + assert_eq!( + result[name][adapter]["body"]["error"], "ssrf_denied", + "{name} {adapter}: {result}" + ); + } + } + assert_eq!( + result["metadata"]["connect"]["body"]["detail"], + "CONNECT 169.254.169.254:80 blocked: declared endpoint check failed" + ); + assert_eq!( + result["metadata"]["forward"]["body"]["detail"], + "GET 169.254.169.254:80 blocked: declared endpoint check failed" + ); + assert_eq!( + result["control_plane"]["connect"]["body"]["detail"], + "CONNECT 203.0.113.10:6443 blocked: allowed_ips check failed" + ); + assert_eq!( + result["outside_allowed_ips"]["forward"]["body"]["detail"], + "GET 203.0.113.10:8080 blocked: allowed_ips check failed" + ); +} + +#[tokio::test] +async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adapters() { + let resolver = SandboxGuard::create(&[ + "--", + "python3", + "-c", + "import socket; print('GATEWAY_IP=' + socket.gethostbyname('host.openshell.internal'))", + ]) + .await + .expect("resolve host gateway inside sandbox"); + let gateway_ip = resolver + .create_output + .lines() + .find_map(|line| line.trim().strip_prefix("GATEWAY_IP=")) + .expect("sandbox gateway IPv4 output") + .to_string(); + gateway_ip + .parse::() + .expect("host gateway must resolve to IPv4 for this Docker e2e"); + + let explicit_server = KeepAliveHttpServer::start() + .await + .expect("start explicit allowed_ips server"); + let implicit_server = KeepAliveHttpServer::start() + .await + .expect("start implicit IP-literal server"); + let policy = + write_ip_literal_success_policy(&gateway_ip, explicit_server.port, implicit_server.port) + .expect("write IP literal policy"); + let policy_path = policy_path(&policy); + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + + for (mode, port) in [ + ("explicit_allowed_ips", explicit_server.port), + ("implicit_ip_literal", implicit_server.port), + ] { + let output = guard + .exec(&["python3", "-c", &proxy_status_script(&gateway_ip, port)]) + .await + .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); + let statuses = parse_json_line(&output); + assert_eq!(statuses["connect"], 200, "{mode} CONNECT: {statuses}"); + assert_eq!(statuses["forward"], 200, "{mode} forward: {statuses}"); + } + + guard.cleanup().await; +} + +#[tokio::test] +async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { + let server = EchoServer::start().await.expect("start TCP echo server"); + let policy = write_policy(TEST_SERVER_HOST, server.port, " tls: skip") + .expect("write tls: skip policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]) + b"not-http-or-tls" + bytes(range(64)) + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{{HOST}}:{{PORT}}" + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += sock.recv(4096) + if int(response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + sock.sendall(PAYLOAD) + echoed = b"" + while len(echoed) < len(PAYLOAD): + chunk = sock.recv(len(PAYLOAD) - len(echoed)) + if not chunk: + break + echoed += chunk + if echoed != PAYLOAD: + raise RuntimeError("opaque payload changed in transit") +print("RAW_RELAY_OK") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + assert!( + guard.create_output.contains("RAW_RELAY_OK"), + "raw relay did not preserve the opaque payload:\n{}", + guard.create_output + ); +} + +#[tokio::test] +async fn middleware_redacts_request_bodies_through_both_adapters() { + let server = RequestBodyEchoServer::start() + .await + .expect("start request body echo server"); + let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") + .expect("write middleware policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import json +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +SECRET = "sk-1234567890abcdef" + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("incomplete response headers") + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + status = int(headers.split(None, 2)[1]) + if status != 200: + raise RuntimeError(f"request failed with HTTP {{status}}: {{body!r}}") + return json.loads(body[:length]) + +def request_bytes(target): + body = json.dumps({{"api_key": SECRET}}, separators=(",", ":")).encode() + return ( + f"POST {{target}} HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {{len(body)}}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + +target = f"{{HOST}}:{{PORT}}" +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as forward_sock: + forward_sock.sendall(request_bytes(f"http://{{target}}/middleware")) + forward = read_response(forward_sock) + +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as connect_sock: + connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + connect_response = b"" + while b"\r\n\r\n" not in connect_response: + connect_response += connect_sock.recv(4096) + if int(connect_response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + connect_sock.sendall(request_bytes("/middleware")) + connect = read_response(connect_sock) + +print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + let result = parse_json_line(&guard.create_output); + for adapter in ["connect", "forward"] { + assert_eq!( + result[adapter]["api_key"], "[REDACTED]", + "{adapter} did not deliver the middleware-transformed body: {result}" + ); + } +} + +#[tokio::test] +async fn fail_closed_middleware_blocks_uninspectable_connect_payload_before_upstream() { + let server = EchoServer::start().await.expect("start TCP echo server"); + let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") + .expect("write fail-closed middleware policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37]) + b"not-http-or-tls" + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{{HOST}}:{{PORT}}" + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += sock.recv(4096) + if int(response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied before tunnel establishment") + sock.sendall(PAYLOAD) + denial = b"" + while True: + try: + chunk = sock.recv(4096) + except ConnectionResetError: + break + if not chunk: + break + denial += chunk + if denial and ( + b"HTTP/1.1 403 Forbidden" not in denial + or b"unsupported_l7_protocol" not in denial + ): + raise RuntimeError(f"missing fail-closed middleware denial: {{denial!r}}") +print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + let output = guard + .exec(&["python3", "-c", &script]) + .await + .expect("exercise uninspectable fail-closed middleware"); + assert!( + output.contains("UNINSPECTABLE_MIDDLEWARE_BLOCKED"), + "uninspectable payload was not blocked:\n{output}" + ); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + server.observed_bytes().is_empty(), + "uninspectable payload reached upstream before middleware denial" + ); + + let logs = run_cli(&[ + "logs", + &guard.name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await + .expect("fetch sandbox logs after middleware denial"); + assert!( + logs.contains("openshell.middleware.traffic_uninspectable") + && logs + .contains("Unsupported tunnel protocol cannot be inspected by required middleware"), + "OCSF logs should explain the fail-closed denial:\n{logs}" + ); + + guard.cleanup().await; +} + +#[tokio::test] +async fn fail_open_middleware_bypasses_uninspectable_tls_skip_connect() { + let server = EchoServer::start().await.expect("start TCP echo server"); + let policy = write_middleware_policy( + TEST_SERVER_HOST, + server.port, + " tls: skip", + "fail_open", + ) + .expect("write fail-open middleware policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80]) + b"middleware-bypass" + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{{HOST}}:{{PORT}}" + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += sock.recv(4096) + if int(response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + sock.sendall(PAYLOAD) + echoed = b"" + while len(echoed) < len(PAYLOAD): + chunk = sock.recv(len(PAYLOAD) - len(echoed)) + if not chunk: + break + echoed += chunk + if echoed != PAYLOAD: + raise RuntimeError(f"fail-open middleware did not preserve raw relay: {{echoed!r}}") +print("UNINSPECTABLE_MIDDLEWARE_BYPASSED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + assert!( + guard + .create_output + .contains("UNINSPECTABLE_MIDDLEWARE_BYPASSED"), + "fail-open middleware did not bypass uninspectable traffic:\n{}", + guard.create_output + ); + assert_eq!( + server.observed_bytes(), + [0x00, 0xff, 0x13, 0x37, 0x80] + .into_iter() + .chain(*b"middleware-bypass") + .collect::>(), + "upstream did not receive the unchanged fail-open payload" + ); +} + +#[tokio::test] +async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { + let server = PipelineProbeServer::start() + .await + .expect("start pipeline probe server"); + let endpoint_options = r#" protocol: rest + enforcement: enforce + rules: + - allow: + method: GET + path: "/allowed""#; + let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) + .expect("write pipeline policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +target = "{host}:{port}" +first = ( + f"GET http://{{target}}/allowed HTTP/1.1\r\n" + f"Host: {{target}}\r\nConnection: keep-alive\r\n\r\n" +) +second = ( + f"POST http://{{target}}/blocked HTTP/1.1\r\n" + f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" +) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + sock.sendall((first + second).encode()) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk +if response.count(b"HTTP/1.1 ") != 1 or b" 200 " not in response.split(b"\r\n", 1)[0]: + raise RuntimeError(f"unexpected pipelined response: {{response!r}}") +print("FORWARD_PIPELINE_CLOSED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + assert!( + guard.create_output.contains("FORWARD_PIPELINE_CLOSED"), + "forward proxy did not close after one response:\n{}", + guard.create_output + ); + + let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); + assert!(observed.starts_with("GET /allowed HTTP/1.1\r\n")); + assert!( + !observed.to_ascii_lowercase().contains("\r\nconnection:"), + "shared relay must remove hop-by-hop connection headers:\n{observed}" + ); + assert!(!observed.contains("/blocked")); +} + +#[tokio::test] +async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters() { + let _provider_lock = PROVIDER_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + delete_provider(PROVIDER_NAME).await; + create_generic_provider(PROVIDER_NAME) + .await + .expect("create generic provider"); + + let result = async { + let server = CredentialProbeServer::start().await?; + let endpoint_options = r#" protocol: rest + enforcement: enforce + request_body_credential_rewrite: true + rules: + - allow: + method: POST + path: "/probe""#; + let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options)?; + let policy_path = policy_path(&policy); + let script = format!( + r#" +import json +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +TOKEN = os.environ[{token_env:?}] + +def proxy_parts(): + proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) + ) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("incomplete response headers") + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + code = int(headers.split(None, 2)[1]) + if code != 200: + raise RuntimeError(f"request failed with HTTP {{code}}") + return json.loads(body[:length]) + +def request_bytes(target): + body = json.dumps({{"credential": TOKEN}}, separators=(",", ":")).encode() + return ( + f"POST {{target}} HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + f"Authorization: Bearer {{TOKEN}}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {{len(body)}}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + +proxy_host, proxy_port = proxy_parts() +target = f"{{HOST}}:{{PORT}}" +with socket.create_connection((proxy_host, proxy_port), timeout=10) as forward_sock: + forward_sock.sendall(request_bytes(f"http://{{target}}/probe")) + forward = read_response(forward_sock) + +with socket.create_connection((proxy_host, proxy_port), timeout=10) as connect_sock: + connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + connect_response = b"" + while b"\r\n\r\n" not in connect_response: + connect_response += connect_sock.recv(4096) + if int(connect_response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + connect_sock.sendall(request_bytes("/probe")) + connect = read_response(connect_sock) + +print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +"#, + host = TEST_SERVER_HOST, + port = server.port, + token_env = TOKEN_ENV, + ); + + SandboxGuard::create(&[ + "--policy", + &policy_path, + "--provider", + PROVIDER_NAME, + "--", + "python3", + "-c", + &script, + ]) + .await + } + .await; + + delete_provider(PROVIDER_NAME).await; + + let guard = result.expect("sandbox create"); + let result = parse_json_line(&guard.create_output); + for adapter in ["connect", "forward"] { + assert_eq!( + result[adapter]["header_resolved"], true, + "{adapter} header placeholder was not resolved: {result}" + ); + assert_eq!( + result[adapter]["body_resolved"], true, + "{adapter} body placeholder was not resolved: {result}" + ); + assert_eq!( + result[adapter]["saw_placeholder"], false, + "{adapter} leaked an unresolved placeholder upstream: {result}" + ); + } + assert!( + !guard.create_output.contains(TEST_SECRET), + "sandbox output exposed the raw provider credential:\n{}", + guard.create_output + ); + assert!( + !guard.create_output.contains(PLACEHOLDER_PREFIX), + "sandbox output exposed an unresolved provider placeholder:\n{}", + guard.create_output + ); +} diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs index 65ba19aa1c..90f0e84024 100644 --- a/e2e/rust/tests/websocket_conformance.rs +++ b/e2e/rust/tests/websocket_conformance.rs @@ -373,7 +373,7 @@ def proxy_parts(): raise RuntimeError(f"invalid proxy URL: {{proxy_url!r}}") return parsed.hostname, parsed.port or 80 -def connect_with_retry(host, port, timeout_seconds=20): +def proxy_socket_with_retry(host, port, mode, timeout_seconds=20): proxy_host, proxy_port = proxy_parts() target = f"{{host}}:{{port}}" deadline = time.monotonic() + timeout_seconds @@ -382,13 +382,14 @@ def connect_with_retry(host, port, timeout_seconds=20): sock = None try: sock = socket.create_connection((proxy_host, proxy_port), timeout=5) - request = f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n" - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if response.startswith("HTTP/1.1 200") or response.startswith("HTTP/1.0 200"): - return sock - first_line = response.splitlines()[0] if response else "" - raise RuntimeError(f"proxy CONNECT failed: {{first_line}}") + if mode == "connect": + request = f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n" + sock.sendall(request.encode("ascii")) + response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") + if not (response.startswith("HTTP/1.1 200") or response.startswith("HTTP/1.0 200")): + first_line = response.splitlines()[0] if response else "" + raise RuntimeError(f"proxy CONNECT failed: {{first_line}}") + return sock except (OSError, RuntimeError) as error: if sock is not None: sock.close() @@ -398,25 +399,28 @@ def connect_with_retry(host, port, timeout_seconds=20): token = os.environ[TOKEN_ENV] payload = json.dumps({{"authorization": "Bearer " + token}}, sort_keys=True) -key = base64.b64encode(os.urandom(16)).decode("ascii") - -with connect_with_retry(HOST, PORT) as sock: - request = ( - f"GET /ws HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\n" - "Sec-WebSocket-Version: 13\r\n" - "\r\n" - ) - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not response.startswith("HTTP/1.1 101"): - raise RuntimeError("websocket upgrade failed") - sock.sendall(masked_text_frame(payload)) - _, response_payload = read_frame(sock) - print(response_payload.decode("utf-8")) +results = {{}} +for mode in ("connect", "forward"): + key = base64.b64encode(os.urandom(16)).decode("ascii") + with proxy_socket_with_retry(HOST, PORT, mode) as sock: + request_target = "/ws" if mode == "connect" else f"http://{{HOST}}:{{PORT}}/ws" + request = ( + f"GET {{request_target}} HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {{key}}\r\n" + "Sec-WebSocket-Version: 13\r\n" + "\r\n" + ) + sock.sendall(request.encode("ascii")) + response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") + if not response.startswith("HTTP/1.1 101"): + raise RuntimeError(f"{{mode}} websocket upgrade failed: {{response!r}}") + sock.sendall(masked_text_frame(payload)) + _, response_payload = read_frame(sock) + results[mode] = json.loads(response_payload.decode("utf-8")) +print(json.dumps(results, sort_keys=True)) "#, host = host, port = port, @@ -425,7 +429,7 @@ with connect_with_retry(HOST, PORT) as sock: } #[tokio::test] -async fn websocket_text_placeholder_is_rewritten_in_sandbox() { +async fn websocket_text_placeholder_is_rewritten_through_both_adapters() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -465,7 +469,14 @@ async fn websocket_text_placeholder_is_rewritten_in_sandbox() { assert!( guard .create_output - .contains(r#"{"saw_placeholder": false, "saw_secret": true}"#), + .contains(r#""connect": {"saw_placeholder": false, "saw_secret": true}"#), + "expected CONNECT upstream to see only the resolved secret marker:\n{}", + guard.create_output + ); + assert!( + guard + .create_output + .contains(r#""forward": {"saw_placeholder": false, "saw_secret": true}"#), "expected upstream to see only the resolved secret marker:\n{}", guard.create_output ); diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 9e32b2d306..c140a0f1e7 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -364,6 +364,10 @@ message GetSandboxConfigResponse { // Operator-registered supervisor middleware services required by the // effective policy. Built-in middleware is not included. repeated SupervisorMiddlewareService supervisor_middleware_services = 9; + // Gateway-configured posture for rejected policy generations. Valid values + // are "fail_closed" and "retain_last_valid". Unknown or empty values must + // be treated as fail_closed by the supervisor. + string policy_validation_failure_mode = 10; } // Connection details for one operator-registered supervisor middleware service.