From 565e337fdfc87af9b45b532cdbd97fb2eb990602 Mon Sep 17 00:00:00 2001 From: Ryan Cook Date: Mon, 25 May 2026 17:11:44 +0100 Subject: [PATCH 1/2] feat: add OCI skill image mounting to AgentRuntime Add kagenti.io/skills annotation on target workload metadata with a JSON array of mounted skill names for downstream discovery (agent card controllers, UI). The annotation is set when the skillImageVolumes feature gate is enabled and removed on skill clearing or AgentRuntime deletion. Assisted-By: Claude (Anthropic AI) Signed-off-by: Ryan Cook Co-authored-by: Cursor --- .../crds/agent.kagenti.dev_agentruntimes.yaml | 55 +++ charts/kagenti-operator/values.yaml | 8 + .../api/v1alpha1/agentruntime_types.go | 43 ++ .../api/v1alpha1/zz_generated.deepcopy.go | 20 + kagenti-operator/cmd/main.go | 1 + .../agent.kagenti.dev_agentruntimes.yaml | 46 ++ .../agent_v1alpha1_agentruntime_full.yaml | 7 +- .../agent_v1alpha1_agentruntime_skills.yaml | 23 + kagenti-operator/docs/api-reference.md | 96 +++- kagenti-operator/docs/architecture.md | 6 +- .../controller/agentruntime_config.go | 18 +- .../controller/agentruntime_controller.go | 83 +++- .../agentruntime_controller_test.go | 94 ++++ .../controller/agentruntime_skills.go | 100 ++++ .../controller/agentruntime_skills_test.go | 309 ++++++++++++ .../webhook/config/feature_gate_loader.go | 2 + .../internal/webhook/config/feature_gates.go | 7 + .../webhook/v1alpha1/agentruntime_webhook.go | 27 ++ .../v1alpha1/agentruntime_webhook_test.go | 102 ++++ kagenti-operator/test/e2e/README.md | 16 +- kagenti-operator/test/e2e/e2e_test.go | 440 ++++++++++++++++++ kagenti-operator/test/e2e/fixtures.go | 145 ++++++ kagenti-operator/test/utils/utils.go | 100 +++- 23 files changed, 1735 insertions(+), 13 deletions(-) create mode 100644 kagenti-operator/config/samples/agent_v1alpha1_agentruntime_skills.yaml create mode 100644 kagenti-operator/internal/controller/agentruntime_skills.go create mode 100644 kagenti-operator/internal/controller/agentruntime_skills_test.go diff --git a/charts/kagenti-operator/crds/agent.kagenti.dev_agentruntimes.yaml b/charts/kagenti-operator/crds/agent.kagenti.dev_agentruntimes.yaml index ff811c0b..29254e16 100644 --- a/charts/kagenti-operator/crds/agent.kagenti.dev_agentruntimes.yaml +++ b/charts/kagenti-operator/crds/agent.kagenti.dev_agentruntimes.yaml @@ -102,6 +102,15 @@ spec: identity: description: Identity specifies optional per-workload identity overrides properties: + allowedAudiences: + description: |- + AllowedAudiences specifies additional JWT audiences that the AuthProxy + sidecar should accept for inbound requests. This is a transitional + mechanism to support application-to-agent flows until the auth model + is finalized. See https://github.com/kagenti/kagenti-operator/issues/368 + items: + type: string + type: array spiffe: description: SPIFFE specifies SPIFFE identity configuration overrides properties: @@ -151,6 +160,52 @@ spec: - permissive - strict type: string + skills: + description: |- + Skills declares OCI skill images to mount into the agent pod as + Kubernetes ImageVolumes. Each skill is mounted read-only at + /agent/skills//. Requires the skillImageVolumes feature gate + and Kubernetes 1.31+ with the ImageVolume feature gate enabled. + items: + description: SkillImageRef identifies an OCI skill image to mount + into the agent pod. + properties: + image: + description: Image is the OCI image reference for the skill. + minLength: 1 + type: string + mountPath: + description: |- + MountPath is the absolute path where the skill image is mounted in + the container. Different agent frameworks expect skills in different + locations (e.g. /agent/skills/my-skill, /app/.claude/skills/my-skill). + minLength: 1 + pattern: ^/.* + type: string + name: + description: |- + Name is a unique identifier for this skill mount, used as the volume + name suffix (skill-). + maxLength: 58 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$ + type: string + pullPolicy: + description: |- + PullPolicy for pulling the OCI skill image. Defaults to Always for + :latest tags and IfNotPresent otherwise (standard Kubernetes behavior). + enum: + - Always + - Never + - IfNotPresent + type: string + required: + - image + - mountPath + - name + type: object + maxItems: 20 + type: array targetRef: description: TargetRef identifies the workload backing this agent runtime (duck typing). diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 2f66a0b2..171f8eae 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -151,6 +151,14 @@ featureGates: # Default false — cached mode is faster and sufficient when namespace ConfigMaps # rarely change. Cache is cleared on webhook pod restart. perWorkloadConfigResolution: false + # combinedSidecar controls whether injection uses a single combined authbridge + # container instead of separate envoy-proxy + spiffe-helper + client-registration. + # Requires the authbridge image. Default false — separate sidecars. + combinedSidecar: false + # skillImageVolumes controls whether AgentRuntime can mount OCI skill images + # as Kubernetes ImageVolumes into agent pods. Requires Kubernetes 1.31+ with + # the ImageVolume feature gate enabled. Default false. + skillImageVolumes: false # Platform defaults for AuthBridge sidecar injection. # These are the lowest-priority layer — overridden by feature gates, diff --git a/kagenti-operator/api/v1alpha1/agentruntime_types.go b/kagenti-operator/api/v1alpha1/agentruntime_types.go index 89db60ac..8d8ba9bc 100644 --- a/kagenti-operator/api/v1alpha1/agentruntime_types.go +++ b/kagenti-operator/api/v1alpha1/agentruntime_types.go @@ -126,6 +126,14 @@ type AgentRuntimeSpec struct { // +optional // +kubebuilder:validation:Enum=disabled;permissive;strict MTLSMode string `json:"mtlsMode,omitempty"` + + // Skills declares OCI skill images to mount into the agent pod as + // Kubernetes ImageVolumes. Each skill is mounted read-only at + // /agent/skills//. Requires the skillImageVolumes feature gate + // and Kubernetes 1.31+ with the ImageVolume feature gate enabled. + // +optional + // +kubebuilder:validation:MaxItems=20 + Skills []SkillImageRef `json:"skills,omitempty"` } // IdentitySpec configures workload identity for an AgentRuntime. @@ -209,6 +217,41 @@ type CardStatus struct { AttestedAgentSpiffeID string `json:"attestedAgentSpiffeID,omitempty"` } +// +kubebuilder:validation:Enum=Always;Never;IfNotPresent +type SkillPullPolicy string + +const ( + SkillPullAlways SkillPullPolicy = "Always" + SkillPullNever SkillPullPolicy = "Never" + SkillPullIfNotPresent SkillPullPolicy = "IfNotPresent" +) + +// SkillImageRef identifies an OCI skill image to mount into the agent pod. +type SkillImageRef struct { + // Name is a unique identifier for this skill mount, used as the volume + // name suffix (skill-). + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=58 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$` + Name string `json:"name"` + + // Image is the OCI image reference for the skill. + // +kubebuilder:validation:MinLength=1 + Image string `json:"image"` + + // MountPath is the absolute path where the skill image is mounted in + // the container. Different agent frameworks expect skills in different + // locations (e.g. /agent/skills/my-skill, /app/.claude/skills/my-skill). + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^/.*` + MountPath string `json:"mountPath"` + + // PullPolicy for pulling the OCI skill image. Defaults to Always for + // :latest tags and IfNotPresent otherwise (standard Kubernetes behavior). + // +optional + PullPolicy SkillPullPolicy `json:"pullPolicy,omitempty"` +} + // AgentRuntimeStatus defines the observed state of AgentRuntime. type AgentRuntimeStatus struct { // Phase is the high-level state of the AgentRuntime diff --git a/kagenti-operator/api/v1alpha1/zz_generated.deepcopy.go b/kagenti-operator/api/v1alpha1/zz_generated.deepcopy.go index 20293e54..7a294f93 100644 --- a/kagenti-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/kagenti-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -382,6 +382,11 @@ func (in *AgentRuntimeSpec) DeepCopyInto(out *AgentRuntimeSpec) { *out = new(TraceSpec) (*in).DeepCopyInto(*out) } + if in.Skills != nil { + in, out := &in.Skills, &out.Skills + *out = make([]SkillImageRef, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentRuntimeSpec. @@ -592,6 +597,21 @@ func (in *SignatureHeader) DeepCopy() *SignatureHeader { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SkillImageRef) DeepCopyInto(out *SkillImageRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SkillImageRef. +func (in *SkillImageRef) DeepCopy() *SkillImageRef { + if in == nil { + return nil + } + out := new(SkillImageRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SkillParameter) DeepCopyInto(out *SkillParameter) { *out = *in diff --git a/kagenti-operator/cmd/main.go b/kagenti-operator/cmd/main.go index 5e1730ea..a6f3f69a 100644 --- a/kagenti-operator/cmd/main.go +++ b/kagenti-operator/cmd/main.go @@ -430,6 +430,7 @@ func main() { Recorder: mgr.GetEventRecorderFor("agentruntime-controller"), EnableCardDiscovery: enableCardDiscovery, SpireTrustDomain: spireTrustDomain, + GetFeatureGates: featureGateLoader.Get, } if enableCardDiscovery { artReconciler.AgentFetcher = agentFetcher diff --git a/kagenti-operator/config/crd/bases/agent.kagenti.dev_agentruntimes.yaml b/kagenti-operator/config/crd/bases/agent.kagenti.dev_agentruntimes.yaml index eb35028a..29254e16 100644 --- a/kagenti-operator/config/crd/bases/agent.kagenti.dev_agentruntimes.yaml +++ b/kagenti-operator/config/crd/bases/agent.kagenti.dev_agentruntimes.yaml @@ -160,6 +160,52 @@ spec: - permissive - strict type: string + skills: + description: |- + Skills declares OCI skill images to mount into the agent pod as + Kubernetes ImageVolumes. Each skill is mounted read-only at + /agent/skills//. Requires the skillImageVolumes feature gate + and Kubernetes 1.31+ with the ImageVolume feature gate enabled. + items: + description: SkillImageRef identifies an OCI skill image to mount + into the agent pod. + properties: + image: + description: Image is the OCI image reference for the skill. + minLength: 1 + type: string + mountPath: + description: |- + MountPath is the absolute path where the skill image is mounted in + the container. Different agent frameworks expect skills in different + locations (e.g. /agent/skills/my-skill, /app/.claude/skills/my-skill). + minLength: 1 + pattern: ^/.* + type: string + name: + description: |- + Name is a unique identifier for this skill mount, used as the volume + name suffix (skill-). + maxLength: 58 + minLength: 1 + pattern: ^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$ + type: string + pullPolicy: + description: |- + PullPolicy for pulling the OCI skill image. Defaults to Always for + :latest tags and IfNotPresent otherwise (standard Kubernetes behavior). + enum: + - Always + - Never + - IfNotPresent + type: string + required: + - image + - mountPath + - name + type: object + maxItems: 20 + type: array targetRef: description: TargetRef identifies the workload backing this agent runtime (duck typing). diff --git a/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_full.yaml b/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_full.yaml index e73e3a0c..23ae2a80 100644 --- a/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_full.yaml +++ b/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_full.yaml @@ -1,5 +1,6 @@ # Full AgentRuntime: enroll a Deployment as an agent with per-workload overrides. -# Overrides the SPIFFE trust domain and configures OTEL trace collection. +# Overrides the SPIFFE trust domain, configures OTEL trace collection, and +# mounts OCI skill images (requires skillImageVolumes feature gate + K8s 1.31+). apiVersion: agent.kagenti.dev/v1alpha1 kind: AgentRuntime metadata: @@ -21,3 +22,7 @@ spec: protocol: grpc sampling: rate: 0.1 + skills: + - name: weather-forecast + image: ghcr.io/redhat-et/skillimage/weather-forecast:v1.0.0 + mountPath: /agent/skills/weather-forecast diff --git a/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_skills.yaml b/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_skills.yaml new file mode 100644 index 00000000..5852bf4b --- /dev/null +++ b/kagenti-operator/config/samples/agent_v1alpha1_agentruntime_skills.yaml @@ -0,0 +1,23 @@ +# AgentRuntime with OCI skill images: mounts skill OCI images as ImageVolumes. +# Requires: skillImageVolumes feature gate enabled, Kubernetes 1.31+. +apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: resume-agent-runtime + namespace: default + labels: + app.kubernetes.io/name: resume-agent +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: resume-agent + skills: + - name: resume-reviewer + image: ghcr.io/redhat-et/skillimage/resume-reviewer:v1.0.0 + mountPath: /agent/skills/resume-reviewer + - name: blog-writer + image: ghcr.io/redhat-et/skillimage/blog-writer:latest + mountPath: /agent/skills/blog-writer + pullPolicy: Always diff --git a/kagenti-operator/docs/api-reference.md b/kagenti-operator/docs/api-reference.md index e427e081..5732870a 100644 --- a/kagenti-operator/docs/api-reference.md +++ b/kagenti-operator/docs/api-reference.md @@ -404,7 +404,7 @@ The controller merges configuration from three layers (highest priority wins): 2. **Namespace defaults** — ConfigMap with `kagenti.io/defaults=true` label in the workload's namespace 3. **Cluster defaults** — `kagenti-platform-config` ConfigMap in `kagenti-system` -> **Note:** Feature gates (`kagenti-feature-gates`) are platform-wide policy and are **not** overrideable by namespace defaults or AgentRuntime CRs. They control which AuthBridge components (envoy proxy, SPIFFE helper, client registration) are enabled globally. +> **Note:** Feature gates (`kagenti-feature-gates`) are platform-wide policy and are **not** overrideable by namespace defaults or AgentRuntime CRs. They control which AuthBridge components (envoy proxy, SPIFFE helper, client registration) are enabled globally, and whether OCI skill image mounting (`skillImageVolumes`) is active. ### Spec Fields @@ -414,6 +414,7 @@ The controller merges configuration from three layers (highest priority wins): | `targetRef` | [TargetRef](#targetref) | Yes | Identifies the workload backing this runtime (uses the same TargetRef type as AgentCard) | | `identity` | [IdentitySpec](#identityspec) | No | Optional per-workload identity overrides | | `trace` | [TraceSpec](#tracespec) | No | Optional per-workload observability overrides | +| `skills` | [][SkillImageRef](#skillimageref) | No | OCI skill images to mount into the agent pod as Kubernetes ImageVolumes. Requires the `skillImageVolumes` feature gate and Kubernetes 1.31+. Max 20 items. | #### IdentitySpec @@ -445,6 +446,59 @@ Configures observability for an AgentRuntime. |-------|------|----------|-------------| | `rate` | float | Yes | Sampling rate (0.0-1.0, inclusive) | +#### SkillImageRef + +Identifies an OCI skill image to mount into the agent pod as a Kubernetes [ImageVolume](https://kubernetes.io/docs/tasks/configure-pod-container/image-volumes/). Skills are packaged as OCI images following the [skillimage](https://github.com/redhat-et/skillimage) convention (`FROM scratch` with `skill.yaml` + `SKILL.md`). + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for this skill mount. Used as the volume name suffix (`skill-`). Must be a valid DNS label (lowercase alphanumeric or hyphens, max 58 characters). | +| `image` | string | Yes | OCI image reference for the skill (e.g., `ghcr.io/redhat-et/skillimage/resume-reviewer:v1.0.0`) | +| `mountPath` | string | Yes | Absolute path where the skill image is mounted in the container. Different agent frameworks expect skills in different locations (e.g., `/agent/skills/my-skill`, `/app/.claude/skills/my-skill`). | +| `pullPolicy` | string | No | Image pull policy: `Always`, `Never`, or `IfNotPresent`. Defaults to `Always` for `:latest` tags, `IfNotPresent` otherwise (standard Kubernetes behavior). | + +**Prerequisites:** +- The `skillImageVolumes` feature gate must be enabled (defaults to `false`) +- Kubernetes 1.31+ with the `ImageVolume` feature gate enabled on the kubelet +- OpenShift 4.18+ (for OpenShift deployments) + +**Behavior:** +- Each skill is mounted as a read-only ImageVolume at the specified `mountPath` +- Skill changes (add, remove, image update, mount path change) trigger rolling updates via config-hash +- Skill volumes use the `skill-` prefix and do not interfere with existing ConfigMap, Secret, or CSI volumes +- The operator sets a `kagenti.io/skills` annotation on the target workload's metadata containing a JSON array of skill names (e.g., `["weather-forecast","resume-reviewer"]`). Downstream systems such as agent card controllers or the Kagenti UI can read this annotation to discover which skills are mounted without inspecting the pod spec. The annotation is removed when skills are cleared or the AgentRuntime is deleted. +- On AgentRuntime deletion, all skill volumes and the `kagenti.io/skills` annotation are removed from the target workload +- If the feature gate is disabled but skills are defined, a `SkillsMounted=False` condition is set with reason `FeatureGateDisabled` + +### Labels and Annotations Applied to Target Workloads + +The AgentRuntime controller applies the following labels and annotations to the target workload: + +**Workload metadata labels:** + +| Label | Value | Description | +|-------|-------|-------------| +| `kagenti.io/type` | `agent` or `tool` | Classifies the workload (from `spec.type`) | +| `app.kubernetes.io/managed-by` | `kagenti-operator` | Indicates the workload is managed by the operator. Removed on AgentRuntime deletion. | + +**Workload metadata annotations:** + +| Annotation | Value | Description | +|------------|-------|-------------| +| `kagenti.io/skills` | JSON array of skill names | Lists mounted skill names (e.g., `["weather-forecast","resume-reviewer"]`). Only set when the `skillImageVolumes` feature gate is enabled and skills are defined. Removed when skills are cleared or on AgentRuntime deletion. | + +**PodTemplateSpec labels:** + +| Label | Value | Description | +|-------|-------|-------------| +| `kagenti.io/type` | `agent` or `tool` | Classifies pods spawned by this workload | + +**PodTemplateSpec annotations:** + +| Annotation | Value | Description | +|------------|-------|-------------| +| `kagenti.io/config-hash` | SHA-256 hex string | Deterministic hash of the resolved configuration. Changes trigger rolling updates. | + ### Status Fields | Field | Type | Description | @@ -464,6 +518,9 @@ Configures observability for an AgentRuntime. | `Ready` | True | `Configured` | Labels and config-hash applied to the target workload | | `Ready` | False | `ConfigHashError` | Failed to compute the config hash | | `Ready` | False | `ConfigApplyError` | Failed to apply labels/annotations to the workload | +| `SkillsMounted` | True | `SkillsApplied` | OCI skill ImageVolumes applied to the target workload | +| `SkillsMounted` | False | `FeatureGateDisabled` | Skills defined but `skillImageVolumes` feature gate is disabled | +| `SkillsMounted` | False | `UnsupportedWorkloadKind` | Skills defined but the target workload kind (e.g., Sandbox) does not support skill ImageVolumes | ### Admission Validation @@ -474,6 +531,8 @@ A validating webhook prevents ownership conflicts: This prevents conflicting label updates where two AgentRuntime CRs with different `type` values (e.g., `agent` vs `tool`) would fight over the same workload's `kagenti.io/type` label. +- **Skill validation**: Duplicate skill names and duplicate mount paths within the same AgentRuntime are rejected. + ### Examples #### Basic Agent Runtime @@ -537,6 +596,41 @@ spec: rate: 1.0 ``` +#### Agent Runtime with OCI Skill Images + +Mount OCI-packaged skills into the agent pod. Requires the `skillImageVolumes` feature gate enabled in the `kagenti-feature-gates` ConfigMap and Kubernetes 1.31+. + +```yaml +apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: resume-agent-runtime + namespace: default +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: resume-agent + skills: + - name: resume-reviewer + image: ghcr.io/redhat-et/skillimage/resume-reviewer:v1.0.0 + mountPath: /agent/skills/resume-reviewer + - name: blog-writer + image: ghcr.io/redhat-et/skillimage/blog-writer:latest + mountPath: /agent/skills/blog-writer + pullPolicy: Always +``` + +To enable the feature gate: + +```yaml +# In the kagenti-feature-gates ConfigMap (kagenti-system namespace) +# or via Helm values: +featureGates: + skillImageVolumes: true +``` + ### kubectl Usage Examples ```bash diff --git a/kagenti-operator/docs/architecture.md b/kagenti-operator/docs/architecture.md index 1ac4a7be..d23a46eb 100644 --- a/kagenti-operator/docs/architecture.md +++ b/kagenti-operator/docs/architecture.md @@ -70,8 +70,10 @@ The Kagenti Operator is a Kubernetes controller that implements the [Operator Pa - Watches AgentRuntime CRs, Deployments, StatefulSets, and ConfigMaps - Applies `kagenti.io/type` label and `kagenti.io/config-hash` annotation to target workloads - Computes config hash from 3-layer merged configuration (cluster defaults → namespace defaults → CR overrides) -- Triggers rolling updates when configuration changes (any layer) -- On CR deletion: preserves type label, updates config-hash to defaults-only, removes managed-by label +- Mounts OCI skill images as Kubernetes ImageVolumes when the `skillImageVolumes` feature gate is enabled (see [SkillImageRef](api-reference.md#skillimageref)) +- Sets `kagenti.io/skills` annotation on target workload metadata with mounted skill names for downstream discovery +- Triggers rolling updates when configuration changes (any layer, including skill additions/removals) +- On CR deletion: preserves type label, updates config-hash to defaults-only, removes managed-by label, skill volumes, and `kagenti.io/skills` annotation - Coordinates with the AuthBridge mutating webhook (in-process) which injects sidecars at Pod CREATE time ### Supporting Components diff --git a/kagenti-operator/internal/controller/agentruntime_config.go b/kagenti-operator/internal/controller/agentruntime_config.go index b1b4ae4c..0688f4f6 100644 --- a/kagenti-operator/internal/controller/agentruntime_config.go +++ b/kagenti-operator/internal/controller/agentruntime_config.go @@ -64,7 +64,6 @@ type resolvedConfig struct { Trace *traceConfig `json:"trace,omitempty"` FeatureGates map[string]string `json:"featureGates,omitempty"` Defaults map[string]string `json:"defaults,omitempty"` - // AuthBridgeMode and MTLSMode change the injected sidecar shape / // transport posture, both of which require a pod restart to take // effect. Including them here folds CR-edit changes into the @@ -89,6 +88,14 @@ type resolvedConfig struct { // formatting / whitespace edits to this CM during peak hours will // trigger a noticeable rollout fan-out. AuthBridgeRuntime string `json:"authBridgeRuntime,omitempty"` + Skills []skillConfig `json:"skills,omitempty"` +} + +type skillConfig struct { + Name string `json:"name"` + Image string `json:"image"` + MountPath string `json:"mountPath"` + PullPolicy string `json:"pullPolicy,omitempty"` } type traceConfig struct { @@ -187,6 +194,15 @@ func resolveConfig(ctx context.Context, c client.Reader, namespace string, spec resolved.AuthBridgeMode = spec.AuthBridgeMode resolved.MTLSMode = spec.MTLSMode + for _, s := range spec.Skills { + resolved.Skills = append(resolved.Skills, skillConfig{ + Name: s.Name, + Image: s.Image, + MountPath: s.MountPath, + PullPolicy: string(s.PullPolicy), + }) + } + return resolved, warnings } diff --git a/kagenti-operator/internal/controller/agentruntime_controller.go b/kagenti-operator/internal/controller/agentruntime_controller.go index e620d1b3..7352aae0 100644 --- a/kagenti-operator/internal/controller/agentruntime_controller.go +++ b/kagenti-operator/internal/controller/agentruntime_controller.go @@ -49,6 +49,7 @@ import ( agentv1alpha1 "github.com/kagenti/operator/api/v1alpha1" "github.com/kagenti/operator/internal/agentcard" "github.com/kagenti/operator/internal/signature" + webhookconfig "github.com/kagenti/operator/internal/webhook/config" ) const ( @@ -57,6 +58,10 @@ const ( // AnnotationConfigHash is the annotation applied to PodTemplateSpec to trigger rolling updates. AnnotationConfigHash = "kagenti.io/config-hash" + // AnnotationSkills is the annotation applied to workload metadata to advertise + // which skill images are mounted. Value is a JSON array of skill names. + AnnotationSkills = "kagenti.io/skills" + // AnnotationRestartPending marks a Sandbox that was scaled to 0 and needs // to be scaled back to 1 on the next reconcile cycle. Two-phase restart // avoids a race with the Sandbox controller's pod-name annotation. @@ -97,6 +102,14 @@ type AgentRuntimeReconciler struct { SignatureProvider signature.Provider EnableCardDiscovery bool SpireTrustDomain string + GetFeatureGates func() *webhookconfig.FeatureGates +} + +func (r *AgentRuntimeReconciler) getFeatureGates() *webhookconfig.FeatureGates { + if r.GetFeatureGates != nil { + return r.GetFeatureGates() + } + return webhookconfig.DefaultFeatureGates() } // +kubebuilder:rbac:groups=agent.kagenti.dev,resources=agentruntimes,verbs=get;list;watch;create;update;patch;delete @@ -210,6 +223,31 @@ func (r *AgentRuntimeReconciler) Reconcile(ctx context.Context, req ctrl.Request return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } + // 6.5. Set SkillsMounted condition based on skills and feature gate state + if len(rt.Spec.Skills) > 0 { + fg := r.getFeatureGates() + if !fg.SkillImageVolumes { + r.setCondition(rt, ConditionTypeSkillsMounted, metav1.ConditionFalse, "FeatureGateDisabled", + "Skills defined but skillImageVolumes feature gate is disabled") + if r.Recorder != nil { + r.Recorder.Event(rt, corev1.EventTypeWarning, "SkillsNotMounted", + "skillImageVolumes feature gate is disabled; enable it to mount OCI skill images") + } + } else if rt.Spec.TargetRef.Kind == KindSandbox { + r.setCondition(rt, ConditionTypeSkillsMounted, metav1.ConditionFalse, "UnsupportedWorkloadKind", + "Sandbox workloads do not support skill ImageVolumes") + if r.Recorder != nil { + r.Recorder.Event(rt, corev1.EventTypeWarning, "SkillsNotMounted", + "Sandbox workloads do not support skill ImageVolumes") + } + } else { + r.setCondition(rt, ConditionTypeSkillsMounted, metav1.ConditionTrue, "SkillsApplied", + fmt.Sprintf("%d skill image(s) mounted", len(rt.Spec.Skills))) + } + } else { + meta.RemoveStatusCondition(&rt.Status.Conditions, ConditionTypeSkillsMounted) + } + // 7. Count configured pods configuredPods, err := r.countConfiguredPods(ctx, rt) if err != nil { @@ -305,6 +343,24 @@ func (r *AgentRuntimeReconciler) applyWorkloadConfig(ctx context.Context, rt *ag workloadLabels[LabelManagedBy] = LabelManagedByValue acc.obj.SetLabels(workloadLabels) + // Advertise mounted skills on workload metadata + workloadAnnotations := acc.obj.GetAnnotations() + if workloadAnnotations == nil { + workloadAnnotations = make(map[string]string) + } + fg := r.getFeatureGates() + if fg.SkillImageVolumes && len(rt.Spec.Skills) > 0 { + names := make([]string, 0, len(rt.Spec.Skills)) + for _, s := range rt.Spec.Skills { + names = append(names, s.Name) + } + b, _ := json.Marshal(names) + workloadAnnotations[AnnotationSkills] = string(b) + } else { + delete(workloadAnnotations, AnnotationSkills) + } + acc.obj.SetAnnotations(workloadAnnotations) + // Apply labels to PodTemplateSpec podLabels := acc.getPodLabels(acc.obj) if podLabels == nil { @@ -321,6 +377,13 @@ func (r *AgentRuntimeReconciler) applyWorkloadConfig(ctx context.Context, rt *ag podAnnotations[AnnotationConfigHash] = configHash acc.setPodAnnotations(acc.obj, podAnnotations) + // Apply skill ImageVolumes when feature gate is enabled + if acc.getPodSpec != nil { + if fg.SkillImageVolumes { + reconcileSkillVolumes(acc.getPodSpec(acc.obj), rt.Spec.Skills) + } + } + logger.Info("Applying config to workload", "workload", ref.Name, "kind", ref.Kind, @@ -580,11 +643,20 @@ func (r *AgentRuntimeReconciler) handleDeletion(ctx context.Context, rt *agentv1 podAnnotations[AnnotationConfigHash] = defaultsHash acc.setPodAnnotations(acc.obj, podAnnotations) - // Remove managed-by from workload metadata + // Remove managed-by label and skills annotation from workload metadata workloadLabels := acc.obj.GetLabels() delete(workloadLabels, LabelManagedBy) acc.obj.SetLabels(workloadLabels) + workloadAnnotations := acc.obj.GetAnnotations() + delete(workloadAnnotations, AnnotationSkills) + acc.obj.SetAnnotations(workloadAnnotations) + + // Remove skill volumes on deletion + if acc.getPodSpec != nil { + reconcileSkillVolumes(acc.getPodSpec(acc.obj), nil) + } + logger.Info("Updated workload to defaults-only config on AgentRuntime deletion", "workload", ref.Name, "kind", ref.Kind) return r.Update(ctx, acc.obj) @@ -1031,13 +1103,14 @@ func (r *AgentRuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { } // runtimePodTemplateAccessor provides uniform access to PodTemplateSpec -// labels and annotations across Deployment and StatefulSet. +// labels, annotations, and PodSpec across Deployment and StatefulSet. type runtimePodTemplateAccessor struct { obj client.Object getPodLabels func(client.Object) map[string]string setPodLabels func(client.Object, map[string]string) getPodAnnotations func(client.Object) map[string]string setPodAnnotations func(client.Object, map[string]string) + getPodSpec func(client.Object) *corev1.PodSpec } func newRuntimePodTemplateAccessor(kind string) (*runtimePodTemplateAccessor, bool) { @@ -1057,6 +1130,9 @@ func newRuntimePodTemplateAccessor(kind string) (*runtimePodTemplateAccessor, bo setPodAnnotations: func(o client.Object, a map[string]string) { o.(*appsv1.Deployment).Spec.Template.Annotations = a }, + getPodSpec: func(o client.Object) *corev1.PodSpec { + return &o.(*appsv1.Deployment).Spec.Template.Spec + }, }, true case "StatefulSet": return &runtimePodTemplateAccessor{ @@ -1073,6 +1149,9 @@ func newRuntimePodTemplateAccessor(kind string) (*runtimePodTemplateAccessor, bo setPodAnnotations: func(o client.Object, a map[string]string) { o.(*appsv1.StatefulSet).Spec.Template.Annotations = a }, + getPodSpec: func(o client.Object) *corev1.PodSpec { + return &o.(*appsv1.StatefulSet).Spec.Template.Spec + }, }, true case KindSandbox: u := &unstructured.Unstructured{} diff --git a/kagenti-operator/internal/controller/agentruntime_controller_test.go b/kagenti-operator/internal/controller/agentruntime_controller_test.go index 357bdc09..73bd0176 100644 --- a/kagenti-operator/internal/controller/agentruntime_controller_test.go +++ b/kagenti-operator/internal/controller/agentruntime_controller_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" agentv1alpha1 "github.com/kagenti/operator/api/v1alpha1" + webhookconfig "github.com/kagenti/operator/internal/webhook/config" ) type stubCardFetcher struct { @@ -158,6 +159,99 @@ var _ = Describe("AgentRuntime Controller", func() { }) }) + Context("When skills annotation is set on workload metadata", func() { + It("should set kagenti.io/skills when feature gate is enabled", func() { + dep := newDeployment("skills-anno-deploy", namespace) + Expect(k8sClient.Create(ctx, dep)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, dep) }() + + rt := newAgentRuntime("skills-anno-rt", namespace, "skills-anno-deploy", agentv1alpha1.RuntimeTypeAgent) + rt.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "weather-forecast", Image: "ghcr.io/example/weather:v1", MountPath: "/agent/skills/weather-forecast"}, + {Name: "resume-reviewer", Image: "ghcr.io/example/resume:v1", MountPath: "/agent/skills/resume-reviewer"}, + } + Expect(k8sClient.Create(ctx, rt)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, rt) }() + + r := newReconciler() + r.GetFeatureGates = func() *webhookconfig.FeatureGates { + return &webhookconfig.FeatureGates{SkillImageVolumes: true} + } + nn := types.NamespacedName{Name: "skills-anno-rt", Namespace: namespace} + + _, _ = r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + Expect(err).NotTo(HaveOccurred()) + + updatedDep := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "skills-anno-deploy", Namespace: namespace}, updatedDep)).To(Succeed()) + + Expect(updatedDep.Annotations).To(HaveKey(AnnotationSkills)) + Expect(updatedDep.Annotations[AnnotationSkills]).To(Equal(`["weather-forecast","resume-reviewer"]`)) + }) + + It("should not set kagenti.io/skills when feature gate is disabled", func() { + dep := newDeployment("skills-anno-off-deploy", namespace) + Expect(k8sClient.Create(ctx, dep)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, dep) }() + + rt := newAgentRuntime("skills-anno-off-rt", namespace, "skills-anno-off-deploy", agentv1alpha1.RuntimeTypeAgent) + rt.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "weather-forecast", Image: "ghcr.io/example/weather:v1", MountPath: "/agent/skills/weather-forecast"}, + } + Expect(k8sClient.Create(ctx, rt)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, rt) }() + + r := newReconciler() + nn := types.NamespacedName{Name: "skills-anno-off-rt", Namespace: namespace} + + _, _ = r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + Expect(err).NotTo(HaveOccurred()) + + updatedDep := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "skills-anno-off-deploy", Namespace: namespace}, updatedDep)).To(Succeed()) + + Expect(updatedDep.Annotations).NotTo(HaveKey(AnnotationSkills)) + }) + + It("should remove kagenti.io/skills on deletion", func() { + dep := newDeployment("skills-anno-del-deploy", namespace) + Expect(k8sClient.Create(ctx, dep)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, dep) }() + + rt := newAgentRuntime("skills-anno-del-rt", namespace, "skills-anno-del-deploy", agentv1alpha1.RuntimeTypeAgent) + rt.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "weather-forecast", Image: "ghcr.io/example/weather:v1", MountPath: "/agent/skills/weather-forecast"}, + } + Expect(k8sClient.Create(ctx, rt)).To(Succeed()) + + r := newReconciler() + r.GetFeatureGates = func() *webhookconfig.FeatureGates { + return &webhookconfig.FeatureGates{SkillImageVolumes: true} + } + nn := types.NamespacedName{Name: "skills-anno-del-rt", Namespace: namespace} + + _, _ = r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + _, _ = r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + + // Verify annotation is present + depBefore := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "skills-anno-del-deploy", Namespace: namespace}, depBefore)).To(Succeed()) + Expect(depBefore.Annotations).To(HaveKey(AnnotationSkills)) + + // Delete AgentRuntime + Expect(k8sClient.Delete(ctx, rt)).To(Succeed()) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: nn}) + Expect(err).NotTo(HaveOccurred()) + + // Verify annotation is removed + depAfter := &appsv1.Deployment{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: "skills-anno-del-deploy", Namespace: namespace}, depAfter)).To(Succeed()) + Expect(depAfter.Annotations).NotTo(HaveKey(AnnotationSkills)) + }) + }) + Context("When setting status", func() { It("should set status to Active with Ready condition", func() { dep := newDeployment("status-deploy", namespace) diff --git a/kagenti-operator/internal/controller/agentruntime_skills.go b/kagenti-operator/internal/controller/agentruntime_skills.go new file mode 100644 index 00000000..2a7542dd --- /dev/null +++ b/kagenti-operator/internal/controller/agentruntime_skills.go @@ -0,0 +1,100 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + + agentv1alpha1 "github.com/kagenti/operator/api/v1alpha1" +) + +const ( + SkillVolumePrefix = "skill-" + + ConditionTypeSkillsMounted = "SkillsMounted" +) + +// reconcileSkillVolumes declaratively reconciles skill ImageVolumes in a PodSpec. +// It adds volumes/mounts for desired skills and removes any stale skill-prefixed +// volumes no longer in the desired list. Pass nil to remove all skill volumes. +func reconcileSkillVolumes(podSpec *corev1.PodSpec, desiredSkills []agentv1alpha1.SkillImageRef) { + desired := make(map[string]agentv1alpha1.SkillImageRef, len(desiredSkills)) + for _, s := range desiredSkills { + desired[SkillVolumePrefix+s.Name] = s + } + + var keptVolumes []corev1.Volume + for _, v := range podSpec.Volumes { + if !strings.HasPrefix(v.Name, SkillVolumePrefix) { + keptVolumes = append(keptVolumes, v) + continue + } + if skill, ok := desired[v.Name]; ok { + keptVolumes = append(keptVolumes, buildSkillVolume(skill)) + delete(desired, v.Name) + } + } + for _, skill := range desiredSkills { + volName := SkillVolumePrefix + skill.Name + if _, ok := desired[volName]; ok { + keptVolumes = append(keptVolumes, buildSkillVolume(skill)) + } + } + podSpec.Volumes = keptVolumes + + desiredMountNames := make(map[string]bool, len(desiredSkills)) + for _, s := range desiredSkills { + desiredMountNames[SkillVolumePrefix+s.Name] = true + } + if len(podSpec.Containers) > 0 { + reconcileSkillMounts(&podSpec.Containers[0], desiredSkills, desiredMountNames) + } +} + +func buildSkillVolume(skill agentv1alpha1.SkillImageRef) corev1.Volume { + return corev1.Volume{ + Name: SkillVolumePrefix + skill.Name, + VolumeSource: corev1.VolumeSource{ + Image: &corev1.ImageVolumeSource{ + Reference: skill.Image, + PullPolicy: corev1.PullPolicy(skill.PullPolicy), + }, + }, + } +} + +func reconcileSkillMounts(container *corev1.Container, desiredSkills []agentv1alpha1.SkillImageRef, desiredMountNames map[string]bool) { + var keptMounts []corev1.VolumeMount + for _, m := range container.VolumeMounts { + if strings.HasPrefix(m.Name, SkillVolumePrefix) && !desiredMountNames[m.Name] { + continue + } + if !strings.HasPrefix(m.Name, SkillVolumePrefix) { + keptMounts = append(keptMounts, m) + } + } + for _, s := range desiredSkills { + keptMounts = append(keptMounts, corev1.VolumeMount{ + Name: SkillVolumePrefix + s.Name, + MountPath: s.MountPath, + ReadOnly: true, + }) + } + container.VolumeMounts = keptMounts +} diff --git a/kagenti-operator/internal/controller/agentruntime_skills_test.go b/kagenti-operator/internal/controller/agentruntime_skills_test.go new file mode 100644 index 00000000..d31c0e39 --- /dev/null +++ b/kagenti-operator/internal/controller/agentruntime_skills_test.go @@ -0,0 +1,309 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + + agentv1alpha1 "github.com/kagenti/operator/api/v1alpha1" +) + +var _ = Describe("Skill Volume Reconciliation", func() { + Context("reconcileSkillVolumes", func() { + It("should add skill volumes to an empty PodSpec", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "agent", Image: "test:latest"}}, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "resume-reviewer", Image: "ghcr.io/example/resume-reviewer:v1.0.0", MountPath: "/agent/skills/resume-reviewer"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].Name).To(Equal("skill-resume-reviewer")) + Expect(podSpec.Volumes[0].VolumeSource.Image).NotTo(BeNil()) + Expect(podSpec.Volumes[0].VolumeSource.Image.Reference).To(Equal("ghcr.io/example/resume-reviewer:v1.0.0")) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(podSpec.Containers[0].VolumeMounts[0].Name).To(Equal("skill-resume-reviewer")) + Expect(podSpec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/agent/skills/resume-reviewer")) + Expect(podSpec.Containers[0].VolumeMounts[0].ReadOnly).To(BeTrue()) + }) + + It("should preserve existing non-skill volumes", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "agent", + Image: "test:latest", + VolumeMounts: []corev1.VolumeMount{ + {Name: "config", MountPath: "/etc/config"}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "my-config"}, + }}}, + }, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "blog-writer", Image: "ghcr.io/example/blog-writer:latest", MountPath: "/app/skills/blog-writer", PullPolicy: agentv1alpha1.SkillPullAlways}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Volumes).To(HaveLen(2)) + Expect(podSpec.Volumes[0].Name).To(Equal("config")) + Expect(podSpec.Volumes[1].Name).To(Equal("skill-blog-writer")) + Expect(podSpec.Volumes[1].VolumeSource.Image.PullPolicy).To(Equal(corev1.PullAlways)) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(2)) + Expect(podSpec.Containers[0].VolumeMounts[0].Name).To(Equal("config")) + Expect(podSpec.Containers[0].VolumeMounts[1].Name).To(Equal("skill-blog-writer")) + Expect(podSpec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/app/skills/blog-writer")) + }) + + It("should remove stale skill volumes", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "agent", + Image: "test:latest", + VolumeMounts: []corev1.VolumeMount{ + {Name: "skill-old-skill", MountPath: "/agent/skills/old-skill", ReadOnly: true}, + {Name: "skill-keep-skill", MountPath: "/agent/skills/keep-skill", ReadOnly: true}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "skill-old-skill", VolumeSource: corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "old:v1"}}}, + {Name: "skill-keep-skill", VolumeSource: corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "keep:v1"}}}, + }, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "keep-skill", Image: "keep:v1", MountPath: "/agent/skills/keep-skill"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].Name).To(Equal("skill-keep-skill")) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(podSpec.Containers[0].VolumeMounts[0].Name).To(Equal("skill-keep-skill")) + }) + + It("should remove all skill volumes when desired is nil", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "agent", + Image: "test:latest", + VolumeMounts: []corev1.VolumeMount{ + {Name: "config", MountPath: "/etc/config"}, + {Name: "skill-a", MountPath: "/agent/skills/a", ReadOnly: true}, + {Name: "skill-b", MountPath: "/agent/skills/b", ReadOnly: true}, + }, + }}, + Volumes: []corev1.Volume{ + {Name: "config", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "my-config"}, + }}}, + {Name: "skill-a", VolumeSource: corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "a:v1"}}}, + {Name: "skill-b", VolumeSource: corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "b:v1"}}}, + }, + } + + reconcileSkillVolumes(podSpec, nil) + + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].Name).To(Equal("config")) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(podSpec.Containers[0].VolumeMounts[0].Name).To(Equal("config")) + }) + + It("should update skill image reference", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "agent", Image: "test:latest"}}, + Volumes: []corev1.Volume{ + {Name: "skill-my-skill", VolumeSource: corev1.VolumeSource{Image: &corev1.ImageVolumeSource{Reference: "old:v1"}}}, + }, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "my-skill", Image: "new:v2", MountPath: "/agent/skills/my-skill"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Volumes).To(HaveLen(1)) + Expect(podSpec.Volumes[0].VolumeSource.Image.Reference).To(Equal("new:v2")) + }) + + It("should only mount skills to the first container", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "agent", Image: "agent:latest"}, + {Name: "sidecar", Image: "sidecar:latest"}, + }, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "skill-a", Image: "a:v1", MountPath: "/app/.claude/skills/skill-a"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(1)) + Expect(podSpec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/app/.claude/skills/skill-a")) + Expect(podSpec.Containers[1].VolumeMounts).To(BeEmpty()) + }) + + It("should not mount skills to injected sidecar containers", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "agent", Image: "my-agent:latest"}, + {Name: "envoy-proxy", Image: "envoyproxy/envoy:v1.30"}, + {Name: "spiffe-helper", Image: "ghcr.io/spiffe/spiffe-helper:latest"}, + {Name: "kagenti-client-registration", Image: "kagenti/client-reg:latest"}, + }, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "resume-reviewer", Image: "ghcr.io/example/resume-reviewer:v1.0.0", MountPath: "/agent/skills/resume-reviewer"}, + {Name: "blog-writer", Image: "ghcr.io/example/blog-writer:latest", MountPath: "/app/.claude/skills/blog-writer"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(2)) + Expect(podSpec.Containers[0].VolumeMounts[0].Name).To(Equal("skill-resume-reviewer")) + Expect(podSpec.Containers[0].VolumeMounts[1].Name).To(Equal("skill-blog-writer")) + for _, sidecar := range podSpec.Containers[1:] { + Expect(sidecar.VolumeMounts).To(BeEmpty(), + "sidecar %q should not have skill mounts", sidecar.Name) + } + }) + + It("should set pull policy when specified", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "agent", Image: "test:latest"}}, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "s1", Image: "img:latest", MountPath: "/skills/s1", PullPolicy: agentv1alpha1.SkillPullAlways}, + {Name: "s2", Image: "img:v1.0.0", MountPath: "/skills/s2", PullPolicy: agentv1alpha1.SkillPullIfNotPresent}, + {Name: "s3", Image: "img:v2.0.0", MountPath: "/skills/s3"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Volumes).To(HaveLen(3)) + Expect(podSpec.Volumes[0].VolumeSource.Image.PullPolicy).To(Equal(corev1.PullAlways)) + Expect(podSpec.Volumes[1].VolumeSource.Image.PullPolicy).To(Equal(corev1.PullIfNotPresent)) + Expect(podSpec.Volumes[2].VolumeSource.Image.PullPolicy).To(Equal(corev1.PullPolicy(""))) + }) + + It("should use different mount paths for different frameworks", func() { + podSpec := &corev1.PodSpec{ + Containers: []corev1.Container{{Name: "agent", Image: "test:latest"}}, + } + skills := []agentv1alpha1.SkillImageRef{ + {Name: "claude-skill", Image: "img:v1", MountPath: "/app/.claude/skills/my-skill"}, + {Name: "cursor-skill", Image: "img:v1", MountPath: "/app/.cursor/rules/my-skill"}, + } + + reconcileSkillVolumes(podSpec, skills) + + Expect(podSpec.Containers[0].VolumeMounts).To(HaveLen(2)) + Expect(podSpec.Containers[0].VolumeMounts[0].MountPath).To(Equal("/app/.claude/skills/my-skill")) + Expect(podSpec.Containers[0].VolumeMounts[1].MountPath).To(Equal("/app/.cursor/rules/my-skill")) + }) + }) + + Context("Config hash includes skills", func() { + ctx := context.Background() + const namespace = "default" + + It("should change when skills are added", func() { + specNoSkills := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-skills"}, + } + specWithSkills := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-skills"}, + Skills: []agentv1alpha1.SkillImageRef{ + {Name: "s1", Image: "img:v1", MountPath: "/skills/s1"}, + }, + } + + r1, _ := ComputeConfigHash(ctx, k8sClient, namespace, specNoSkills) + r2, _ := ComputeConfigHash(ctx, k8sClient, namespace, specWithSkills) + Expect(r1.Hash).NotTo(Equal(r2.Hash)) + }) + + It("should change when skill image changes", func() { + spec1 := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-img"}, + Skills: []agentv1alpha1.SkillImageRef{{Name: "s1", Image: "img:v1", MountPath: "/skills/s1"}}, + } + spec2 := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-img"}, + Skills: []agentv1alpha1.SkillImageRef{{Name: "s1", Image: "img:v2", MountPath: "/skills/s1"}}, + } + + r1, _ := ComputeConfigHash(ctx, k8sClient, namespace, spec1) + r2, _ := ComputeConfigHash(ctx, k8sClient, namespace, spec2) + Expect(r1.Hash).NotTo(Equal(r2.Hash)) + }) + + It("should change when skill pull policy changes", func() { + spec1 := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-pp"}, + Skills: []agentv1alpha1.SkillImageRef{{Name: "s1", Image: "img:v1", MountPath: "/skills/s1", PullPolicy: agentv1alpha1.SkillPullAlways}}, + } + spec2 := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-pp"}, + Skills: []agentv1alpha1.SkillImageRef{{Name: "s1", Image: "img:v1", MountPath: "/skills/s1", PullPolicy: agentv1alpha1.SkillPullIfNotPresent}}, + } + + r1, _ := ComputeConfigHash(ctx, k8sClient, namespace, spec1) + r2, _ := ComputeConfigHash(ctx, k8sClient, namespace, spec2) + Expect(r1.Hash).NotTo(Equal(r2.Hash)) + }) + + It("should change when skill mount path changes", func() { + spec1 := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-mp"}, + Skills: []agentv1alpha1.SkillImageRef{{Name: "s1", Image: "img:v1", MountPath: "/agent/skills/s1"}}, + } + spec2 := &agentv1alpha1.AgentRuntimeSpec{ + Type: agentv1alpha1.RuntimeTypeAgent, + TargetRef: agentv1alpha1.TargetRef{APIVersion: "apps/v1", Kind: "Deployment", Name: "hash-mp"}, + Skills: []agentv1alpha1.SkillImageRef{{Name: "s1", Image: "img:v1", MountPath: "/app/.claude/skills/s1"}}, + } + + r1, _ := ComputeConfigHash(ctx, k8sClient, namespace, spec1) + r2, _ := ComputeConfigHash(ctx, k8sClient, namespace, spec2) + Expect(r1.Hash).NotTo(Equal(r2.Hash)) + }) + }) +}) diff --git a/kagenti-operator/internal/webhook/config/feature_gate_loader.go b/kagenti-operator/internal/webhook/config/feature_gate_loader.go index 54ad071d..c0d3eb3f 100644 --- a/kagenti-operator/internal/webhook/config/feature_gate_loader.go +++ b/kagenti-operator/internal/webhook/config/feature_gate_loader.go @@ -180,6 +180,8 @@ func logFeatureGates(fg *FeatureGates, source string) { "envoyProxy", fg.EnvoyProxy, "injectTools", fg.InjectTools, "perWorkloadConfigResolution", fg.PerWorkloadConfigResolution, + "combinedSidecar", fg.CombinedSidecar, + "skillImageVolumes", fg.SkillImageVolumes, ) log.Info("=============================================") } diff --git a/kagenti-operator/internal/webhook/config/feature_gates.go b/kagenti-operator/internal/webhook/config/feature_gates.go index f6e8eb56..48aec8ca 100644 --- a/kagenti-operator/internal/webhook/config/feature_gates.go +++ b/kagenti-operator/internal/webhook/config/feature_gates.go @@ -22,6 +22,13 @@ type FeatureGates struct { // true → resolved path: webhook reads namespace ConfigMaps at // admission time and injects literal env var values. PerWorkloadConfigResolution bool `json:"perWorkloadConfigResolution" yaml:"perWorkloadConfigResolution"` + // CombinedSidecar controls whether injection uses a single combined authbridge + // container instead of separate envoy-proxy + spiffe-helper + client-registration sidecars. + CombinedSidecar bool `json:"combinedSidecar" yaml:"combinedSidecar"` + // SkillImageVolumes controls whether the AgentRuntime controller mounts OCI + // skill images as Kubernetes ImageVolumes into agent pods. Requires + // Kubernetes 1.31+ with the ImageVolume feature gate enabled. + SkillImageVolumes bool `json:"skillImageVolumes" yaml:"skillImageVolumes"` } // DefaultFeatureGates returns feature gates with sidecar injection enabled for diff --git a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go index 395783e1..8d097b9f 100644 --- a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go +++ b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go @@ -52,6 +52,9 @@ func (v *AgentRuntimeValidator) ValidateCreate(ctx context.Context, rt *agentv1a if err := checkMTLSCompatibleWithMode(rt); err != nil { return nil, err } + if err := validateSkills(rt.Spec.Skills); err != nil { + return nil, err + } return nil, nil } @@ -65,6 +68,9 @@ func (v *AgentRuntimeValidator) ValidateUpdate(ctx context.Context, _ *agentv1al if err := checkMTLSCompatibleWithMode(rt); err != nil { return nil, err } + if err := validateSkills(rt.Spec.Skills); err != nil { + return nil, err + } return nil, nil } @@ -134,3 +140,24 @@ func (v *AgentRuntimeValidator) checkDuplicateTargetRef(ctx context.Context, rt return nil } + +func validateSkills(skills []agentv1alpha1.SkillImageRef) error { + if len(skills) == 0 { + return nil + } + + seenNames := make(map[string]bool, len(skills)) + seenPaths := make(map[string]bool, len(skills)) + for i, skill := range skills { + if seenNames[skill.Name] { + return fmt.Errorf("spec.skills[%d]: duplicate skill name %q", i, skill.Name) + } + seenNames[skill.Name] = true + + if seenPaths[skill.MountPath] { + return fmt.Errorf("spec.skills[%d]: duplicate mountPath %q", i, skill.MountPath) + } + seenPaths[skill.MountPath] = true + } + return nil +} diff --git a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go index a315a4a3..6831b473 100644 --- a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go +++ b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go @@ -310,3 +310,105 @@ func TestAgentRuntimeValidator_MTLSCompatWithMode(t *testing.T) { }) } } + +func TestValidateSkills(t *testing.T) { + t.Run("empty skills is valid", func(t *testing.T) { + if err := validateSkills(nil); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("valid skills succeed", func(t *testing.T) { + skills := []agentv1alpha1.SkillImageRef{ + {Name: "resume-reviewer", Image: "ghcr.io/example/resume:v1", MountPath: "/agent/skills/resume-reviewer"}, + {Name: "blog-writer", Image: "ghcr.io/example/blog:v1", MountPath: "/agent/skills/blog-writer"}, + } + if err := validateSkills(skills); err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("duplicate names rejected", func(t *testing.T) { + skills := []agentv1alpha1.SkillImageRef{ + {Name: "my-skill", Image: "img:v1", MountPath: "/skills/a"}, + {Name: "my-skill", Image: "img:v2", MountPath: "/skills/b"}, + } + err := validateSkills(skills) + if err == nil { + t.Fatal("expected error for duplicate skill names") + } + if !strings.Contains(err.Error(), "duplicate skill name") { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("duplicate mountPath rejected", func(t *testing.T) { + skills := []agentv1alpha1.SkillImageRef{ + {Name: "skill-a", Image: "img:v1", MountPath: "/agent/skills/shared"}, + {Name: "skill-b", Image: "img:v2", MountPath: "/agent/skills/shared"}, + } + err := validateSkills(skills) + if err == nil { + t.Fatal("expected error for duplicate mountPath") + } + if !strings.Contains(err.Error(), "duplicate mountPath") { + t.Errorf("unexpected error message: %v", err) + } + }) +} + +func TestValidateSkills_CreateIntegration(t *testing.T) { + ctx := context.Background() + + t.Run("create with valid skills succeeds", func(t *testing.T) { + v := &AgentRuntimeValidator{} + rt := validAgentRuntime() + rt.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "s1", Image: "img:v1", MountPath: "/skills/s1"}, + } + _, err := v.ValidateCreate(ctx, rt) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("create with duplicate skills rejected", func(t *testing.T) { + v := &AgentRuntimeValidator{} + rt := validAgentRuntime() + rt.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "s1", Image: "img:v1", MountPath: "/skills/a"}, + {Name: "s1", Image: "img:v2", MountPath: "/skills/b"}, + } + _, err := v.ValidateCreate(ctx, rt) + if err == nil { + t.Fatal("expected error for duplicate skill names") + } + }) + + t.Run("create with duplicate mountPath rejected", func(t *testing.T) { + v := &AgentRuntimeValidator{} + rt := validAgentRuntime() + rt.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "s1", Image: "img:v1", MountPath: "/skills/shared"}, + {Name: "s2", Image: "img:v2", MountPath: "/skills/shared"}, + } + _, err := v.ValidateCreate(ctx, rt) + if err == nil { + t.Fatal("expected error for duplicate mountPath") + } + }) + + t.Run("update with duplicate skills rejected", func(t *testing.T) { + v := &AgentRuntimeValidator{} + old := validAgentRuntime() + updated := validAgentRuntime() + updated.Spec.Skills = []agentv1alpha1.SkillImageRef{ + {Name: "s1", Image: "img:v1", MountPath: "/skills/a"}, + {Name: "s1", Image: "img:v2", MountPath: "/skills/b"}, + } + _, err := v.ValidateUpdate(ctx, old, updated) + if err == nil { + t.Fatal("expected error for duplicate skill names on update") + } + }) +} diff --git a/kagenti-operator/test/e2e/README.md b/kagenti-operator/test/e2e/README.md index ef81d4eb..5dcf2da8 100644 --- a/kagenti-operator/test/e2e/README.md +++ b/kagenti-operator/test/e2e/README.md @@ -1,11 +1,13 @@ # E2E Tests -End-to-end tests for the kagenti-operator. The suite runs 20 specs: +End-to-end tests for the kagenti-operator. The suite runs 32 specs: - **Manager tests** (2 specs) — controller pod readiness and Prometheus metrics - **AuthBridge Injection tests** (4 specs) — sidecar injection, idempotency, opt-out, and HTTP validation - **AgentCard tests** (6 specs) — webhook validation, auto-discovery, duplicate prevention, audit mode, and SPIRE signature verification - **AgentRuntime tests** (8 specs) — label application, status lifecycle, idempotency, error handling, tool type, StatefulSet support, identity/trace overrides, and deletion cleanup +- **Combined tests** (5 specs) — AgentRuntime + AgentCard + Auth Bridge integration: labels, auto-created card, sidecar injection, identity binding, and deletion cleanup +- **Skill Image Volumes tests** (7 specs) — feature gate disabled/enabled, volume mounting, update, removal, deletion cleanup, and webhook validation ## Prerequisites @@ -22,7 +24,7 @@ The test suite auto-detects Docker vs Podman. No env vars needed. AuthBridge sid # Create a fresh Kind cluster kind delete cluster 2>/dev/null; kind create cluster -# Run all 20 specs (~15 min) +# Run all 32 specs (~18 min) make test-e2e ``` @@ -56,6 +58,9 @@ go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="Manager" # AgentRuntime tests only (~3 min) go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="AgentRuntime E2E" + +# Skill Image Volumes tests only (~4 min) +go test ./test/e2e/ -v -ginkgo.v -ginkgo.focus="Skill Image Volumes" ``` ## Cleanup @@ -86,6 +91,13 @@ kind delete cluster | Tool type label | Tool type | AgentRuntime with type=tool applies `kagenti.io/type=tool` label and no AgentCard is created | | StatefulSet target | StatefulSet target | AgentRuntime applies labels, config-hash, and reaches Active for a StatefulSet workload | | Identity/trace overrides | Identity and trace overrides | AgentRuntime with identity+trace spec produces a different config-hash than a minimal CR | +| Feature gate disabled | Skill Image Volumes | Skills declared but SkillsMounted=False with reason FeatureGateDisabled; no skill volumes on Deployment | +| Mount skill ImageVolumes | Skill Image Volumes (enabled) | AgentRuntime with skills adds Image volumes, read-only mounts, SkillsMounted=True, config-hash, and `kagenti.io/skills` annotation | +| Update skill image | Skill Image Volumes (enabled) | Changing skill image reference updates Deployment volume and config-hash | +| Remove skills | Skill Image Volumes (enabled) | Removing skills from CR removes all skill volumes, mounts, and `kagenti.io/skills` annotation from Deployment | +| Skill deletion cleanup | Skill Image Volumes (enabled) | AgentRuntime deletion removes skill volumes and `kagenti.io/skills` annotation from target Deployment | +| Duplicate skill names | Webhook validation | Webhook rejects AgentRuntime with duplicate skill names | +| Duplicate skill mountPaths | Webhook validation | Webhook rejects AgentRuntime with duplicate skill mountPaths | ## Architecture diff --git a/kagenti-operator/test/e2e/e2e_test.go b/kagenti-operator/test/e2e/e2e_test.go index 7833a924..e27e8b4f 100644 --- a/kagenti-operator/test/e2e/e2e_test.go +++ b/kagenti-operator/test/e2e/e2e_test.go @@ -1940,3 +1940,443 @@ rules: }).Should(Succeed()) }) }) + +var _ = Describe("Skill Image Volumes E2E", Ordered, func() { + const controllerNamespace = "kagenti-operator-system" + const controllerDeployment = "kagenti-operator-controller-manager" + + BeforeAll(func() { + By("ensuring mlflow-operator ClusterRole exists for ServiceAccount informer") + clusterRoleCmd := exec.Command("kubectl", "apply", "-f", "-") + clusterRoleCmd.Stdin = strings.NewReader(`apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mlflow-operator-mlflow-integration +rules: +- apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["list", "watch"] +`) + _, _ = utils.Run(clusterRoleCmd) + + Expect(utils.DeployController(controllerNamespace, projectImage)).To(Succeed(), "Failed to deploy controller") + + By("waiting for controller-manager to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", + "-n", controllerNamespace, + "-o", "go-template={{ range .items }}{{ if not .metadata.deletionTimestamp }}{{ .status.phase }}{{ end }}{{ end }}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Running")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("waiting for webhook endpoint to be ready") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", + "kagenti-operator-webhook-service", "-n", controllerNamespace, + "-o", "jsonpath={.subsets[0].addresses[0].ip}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("creating skill test namespace") + cmd := exec.Command("kubectl", "create", "ns", skillTestNamespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", skillTestNamespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("ensuring kagenti-system namespace exists") + cmd = exec.Command("kubectl", "create", "ns", "kagenti-system") + _, _ = utils.Run(cmd) + + By("creating cluster defaults ConfigMap") + _, err = utils.KubectlApplyStdin(runtimeClusterDefaultsConfigMapFixture(), "kagenti-system") + Expect(err).NotTo(HaveOccurred()) + + By("deploying skill agent target workload") + _, err = utils.KubectlApplyStdin(skillTargetDeploymentFixture(), skillTestNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(utils.WaitForDeploymentReady("skill-agent-target", skillTestNamespace, 2*time.Minute)).To(Succeed()) + }) + + AfterAll(func() { + By("deleting skill test namespace") + cmd := exec.Command("kubectl", "delete", "ns", skillTestNamespace, "--ignore-not-found") + _, _ = utils.Run(cmd) + + By("cleaning up cluster defaults ConfigMap") + cmd = exec.Command("kubectl", "delete", "configmap", "kagenti-platform-config", + "-n", "kagenti-system", "--ignore-not-found") + _, _ = utils.Run(cmd) + + By("cleaning up feature-gates ConfigMap") + cmd = exec.Command("kubectl", "delete", "configmap", "kagenti-feature-gates", + "-n", controllerNamespace, "--ignore-not-found") + _, _ = utils.Run(cmd) + + By("cleaning up mlflow-operator ClusterRole") + cmd = exec.Command("kubectl", "delete", "clusterrole", + "mlflow-operator-mlflow-integration", "--ignore-not-found") + _, _ = utils.Run(cmd) + + utils.UndeployController() + }) + + AfterEach(func() { + if CurrentSpecReport().Failed() { + for _, diag := range []struct { + label string + args []string + }{ + {"Controller logs", []string{ + "logs", "-l", "control-plane=controller-manager", + "-n", controllerNamespace, "--tail=100", + }}, + {"Events", []string{"get", "events", "-n", skillTestNamespace, "--sort-by=.lastTimestamp"}}, + {"AgentRuntimes", []string{"get", "agentruntimes", "-n", skillTestNamespace, "-o", "yaml"}}, + {"Deployments", []string{"get", "deployments", "-n", skillTestNamespace, "-o", "yaml"}}, + } { + cmd := exec.Command("kubectl", diag.args...) + out, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "%s:\n%s\n", diag.label, out) + } + } + } + }) + + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + + Context("Feature gate disabled (default)", Ordered, func() { + It("should set SkillsMounted=False when feature gate is disabled", func() { + By("creating AgentRuntime with skills (feature gate disabled)") + Eventually(func() error { + _, err := utils.KubectlApplyStdin(skillAgentRuntimeFixture(), skillTestNamespace) + return err + }, 1*time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for AgentRuntime phase=Active") + Eventually(func(g Gomega) { + phase, err := utils.KubectlGetJsonpath("agentruntime", "skill-agent-runtime", + skillTestNamespace, "{.status.phase}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(phase).To(Equal("Active")) + }).Should(Succeed()) + + By("verifying SkillsMounted=False with reason FeatureGateDisabled") + Eventually(func(g Gomega) { + status, err := utils.KubectlGetJsonpath("agentruntime", "skill-agent-runtime", + skillTestNamespace, + "{.status.conditions[?(@.type=='SkillsMounted')].status}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(status).To(Equal("False")) + + reason, err := utils.KubectlGetJsonpath("agentruntime", "skill-agent-runtime", + skillTestNamespace, + "{.status.conditions[?(@.type=='SkillsMounted')].reason}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(reason).To(Equal("FeatureGateDisabled")) + }).Should(Succeed()) + + By("verifying NO skill volumes on Deployment spec") + Eventually(func(g Gomega) { + volumes, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.volumes[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(volumes).NotTo(ContainSubstring("skill-")) + }).Should(Succeed()) + + By("cleaning up AgentRuntime for next context") + cmd := exec.Command("kubectl", "delete", "agentruntime", "skill-agent-runtime", + "-n", skillTestNamespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "agentruntime", "skill-agent-runtime", + "-n", skillTestNamespace) + _, err := cmd.CombinedOutput() + g.Expect(err).To(HaveOccurred(), "AgentRuntime should be deleted") + }).Should(Succeed()) + }) + }) + + Context("Feature gate enabled", Ordered, func() { + var initialConfigHash string + + BeforeAll(func() { + By("enabling skillImageVolumes feature gate") + Expect(utils.EnableSkillImageVolumes(controllerNamespace, controllerDeployment)).To(Succeed()) + + By("waiting for controller to be ready after feature gate patch") + goTmpl := "{{ range .items }}" + + "{{ if not .metadata.deletionTimestamp }}" + + "{{ .status.phase }}{{ end }}{{ end }}" + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", + "-l", "control-plane=controller-manager", + "-n", controllerNamespace, + "-o", "go-template="+goTmpl) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Running")) + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + + By("waiting for webhook endpoint to be ready after restart") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", + "kagenti-operator-webhook-service", "-n", controllerNamespace, + "-o", "jsonpath={.subsets[0].addresses[0].ip}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(BeEmpty(), "webhook endpoint not yet populated") + }, 2*time.Minute, 2*time.Second).Should(Succeed()) + }) + + It("should mount skill ImageVolumes to target Deployment", func() { + By("creating AgentRuntime with 2 skills") + Eventually(func() error { + _, err := utils.KubectlApplyStdin(skillAgentRuntimeFixture(), skillTestNamespace) + return err + }, 1*time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for AgentRuntime phase=Active") + Eventually(func(g Gomega) { + phase, err := utils.KubectlGetJsonpath("agentruntime", "skill-agent-runtime", + skillTestNamespace, "{.status.phase}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(phase).To(Equal("Active")) + }).Should(Succeed()) + + By("verifying skill volumes on Deployment spec") + Eventually(func(g Gomega) { + volumes, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.volumes[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(volumes).To(ContainSubstring("skill-resume-reviewer")) + g.Expect(volumes).To(ContainSubstring("skill-blog-writer")) + }).Should(Succeed()) + + By("verifying skill volume image references") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "skill-agent-target", + "-n", skillTestNamespace, + "-o", "jsonpath={.spec.template.spec.volumes}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("registry.k8s.io/pause:3.9")) + g.Expect(output).To(ContainSubstring("registry.k8s.io/pause:3.10")) + }).Should(Succeed()) + + By("verifying skill volume mounts on agent container") + Eventually(func(g Gomega) { + mounts, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.containers[0].volumeMounts[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(mounts).To(ContainSubstring("skill-resume-reviewer")) + g.Expect(mounts).To(ContainSubstring("skill-blog-writer")) + }).Should(Succeed()) + + By("verifying skill mount paths") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "skill-agent-target", + "-n", skillTestNamespace, + "-o", "jsonpath={.spec.template.spec.containers[0].volumeMounts}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("/agent/skills/resume-reviewer")) + g.Expect(output).To(ContainSubstring("/agent/skills/blog-writer")) + }).Should(Succeed()) + + By("verifying SkillsMounted=True condition") + Eventually(func(g Gomega) { + status, err := utils.KubectlGetJsonpath("agentruntime", "skill-agent-runtime", + skillTestNamespace, + "{.status.conditions[?(@.type=='SkillsMounted')].status}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(status).To(Equal("True")) + + reason, err := utils.KubectlGetJsonpath("agentruntime", "skill-agent-runtime", + skillTestNamespace, + "{.status.conditions[?(@.type=='SkillsMounted')].reason}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(reason).To(Equal("SkillsApplied")) + }).Should(Succeed()) + + By("verifying kagenti.io/skills annotation on Deployment metadata") + Eventually(func(g Gomega) { + ann, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.metadata.annotations['kagenti\\.io/skills']}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ann).To(ContainSubstring("resume-reviewer")) + g.Expect(ann).To(ContainSubstring("blog-writer")) + }).Should(Succeed()) + + By("recording initial config-hash") + Eventually(func(g Gomega) { + hash, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.metadata.annotations['kagenti\\.io/config-hash']}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(hash).To(HaveLen(64)) + initialConfigHash = hash + }).Should(Succeed()) + }) + + It("should update volumes when skill image changes", func() { + By("updating AgentRuntime with changed skill image") + _, err := utils.KubectlApplyStdin(skillAgentRuntimeUpdatedFixture(), skillTestNamespace) + Expect(err).NotTo(HaveOccurred()) + + By("verifying Deployment spec has new image reference") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "deployment", "skill-agent-target", + "-n", skillTestNamespace, + "-o", "jsonpath={.spec.template.spec.volumes}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).NotTo(ContainSubstring("registry.k8s.io/pause:3.9")) + g.Expect(output).To(ContainSubstring("registry.k8s.io/pause:3.10")) + }).Should(Succeed()) + + By("verifying config-hash changed") + Eventually(func(g Gomega) { + hash, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.metadata.annotations['kagenti\\.io/config-hash']}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(hash).To(HaveLen(64)) + g.Expect(hash).NotTo(Equal(initialConfigHash)) + initialConfigHash = hash + }).Should(Succeed()) + }) + + It("should remove skill volumes when skills removed from CR", func() { + By("updating AgentRuntime to remove all skills") + _, err := utils.KubectlApplyStdin(skillAgentRuntimeNoSkillsFixture(), skillTestNamespace) + Expect(err).NotTo(HaveOccurred()) + + By("verifying no skill volumes remain on Deployment") + Eventually(func(g Gomega) { + volumes, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.volumes[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(volumes).NotTo(ContainSubstring("skill-")) + }).Should(Succeed()) + + By("verifying no skill mounts remain on container") + Eventually(func(g Gomega) { + mounts, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.containers[0].volumeMounts[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(mounts).NotTo(ContainSubstring("skill-")) + }).Should(Succeed()) + + By("verifying kagenti.io/skills annotation removed from Deployment") + Eventually(func(g Gomega) { + ann, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.metadata.annotations['kagenti\\.io/skills']}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ann).To(BeEmpty()) + }).Should(Succeed()) + + By("verifying config-hash changed again") + Eventually(func(g Gomega) { + hash, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.metadata.annotations['kagenti\\.io/config-hash']}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(hash).To(HaveLen(64)) + g.Expect(hash).NotTo(Equal(initialConfigHash)) + initialConfigHash = hash + }).Should(Succeed()) + }) + + It("should clean up skill volumes on AgentRuntime deletion", func() { + By("re-applying AgentRuntime with skills") + Eventually(func() error { + _, err := utils.KubectlApplyStdin(skillAgentRuntimeFixture(), skillTestNamespace) + return err + }, 1*time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for skill volumes to appear") + Eventually(func(g Gomega) { + volumes, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.volumes[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(volumes).To(ContainSubstring("skill-resume-reviewer")) + }).Should(Succeed()) + + By("deleting the AgentRuntime CR") + cmd := exec.Command("kubectl", "delete", "agentruntime", "skill-agent-runtime", + "-n", skillTestNamespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred()) + + By("verifying AgentRuntime CR is gone") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "agentruntime", "skill-agent-runtime", + "-n", skillTestNamespace) + _, err := cmd.CombinedOutput() + g.Expect(err).To(HaveOccurred(), "AgentRuntime should be deleted") + }).Should(Succeed()) + + By("verifying skill volumes removed from Deployment") + Eventually(func(g Gomega) { + volumes, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.spec.template.spec.volumes[*].name}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(volumes).NotTo(ContainSubstring("skill-")) + }).Should(Succeed()) + + By("verifying kagenti.io/skills annotation removed after deletion") + Eventually(func(g Gomega) { + ann, err := utils.KubectlGetJsonpath("deployment", "skill-agent-target", + skillTestNamespace, + "{.metadata.annotations['kagenti\\.io/skills']}") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(ann).To(BeEmpty()) + }).Should(Succeed()) + }) + }) + + Context("Webhook validation", func() { + It("should reject AgentRuntime with duplicate skill names", func() { + By("attempting to apply AgentRuntime with duplicate skill names") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "apply", "-f", "-", "-n", skillTestNamespace) + cmd.Stdin = strings.NewReader(skillDuplicateNamesAgentRuntimeFixture()) + output, err := cmd.CombinedOutput() + g.Expect(err).To(HaveOccurred(), "kubectl apply should fail for duplicate skill names") + g.Expect(string(output)).To(ContainSubstring("duplicate skill name")) + }, 1*time.Minute, 2*time.Second).Should(Succeed()) + }) + + It("should reject AgentRuntime with duplicate skill mountPaths", func() { + By("attempting to apply AgentRuntime with duplicate mountPaths") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "apply", "-f", "-", "-n", skillTestNamespace) + cmd.Stdin = strings.NewReader(skillDuplicateMountPathAgentRuntimeFixture()) + output, err := cmd.CombinedOutput() + g.Expect(err).To(HaveOccurred(), "kubectl apply should fail for duplicate mountPaths") + g.Expect(string(output)).To(ContainSubstring("duplicate mountPath")) + }, 1*time.Minute, 2*time.Second).Should(Succeed()) + }) + }) +}) diff --git a/kagenti-operator/test/e2e/fixtures.go b/kagenti-operator/test/e2e/fixtures.go index 5cf0ce6b..199bfd3c 100644 --- a/kagenti-operator/test/e2e/fixtures.go +++ b/kagenti-operator/test/e2e/fixtures.go @@ -1370,3 +1370,148 @@ spec: kagenti-enabled: "true" ` } + +// --- Skill Image Volumes E2E fixtures --- + +const skillTestNamespace = "e2e-skills-test" + +func skillTargetDeploymentFixture() string { + return `apiVersion: apps/v1 +kind: Deployment +metadata: + name: skill-agent-target + namespace: ` + skillTestNamespace + ` + labels: + app.kubernetes.io/name: skill-agent-target +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: skill-agent-target + template: + metadata: + labels: + app.kubernetes.io/name: skill-agent-target + kagenti.io/inject: disabled + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: agent + image: registry.k8s.io/pause:3.9 + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL +` +} + +func skillAgentRuntimeFixture() string { + return `apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: skill-agent-runtime + namespace: ` + skillTestNamespace + ` +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: skill-agent-target + skills: + - name: resume-reviewer + image: registry.k8s.io/pause:3.9 + mountPath: /agent/skills/resume-reviewer + - name: blog-writer + image: registry.k8s.io/pause:3.10 + mountPath: /agent/skills/blog-writer + pullPolicy: Always +` +} + +func skillAgentRuntimeUpdatedFixture() string { + return `apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: skill-agent-runtime + namespace: ` + skillTestNamespace + ` +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: skill-agent-target + skills: + - name: resume-reviewer + image: registry.k8s.io/pause:3.10 + mountPath: /agent/skills/resume-reviewer + - name: blog-writer + image: registry.k8s.io/pause:3.10 + mountPath: /agent/skills/blog-writer + pullPolicy: Always +` +} + +func skillAgentRuntimeNoSkillsFixture() string { + return `apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: skill-agent-runtime + namespace: ` + skillTestNamespace + ` +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: skill-agent-target +` +} + +func skillDuplicateNamesAgentRuntimeFixture() string { + return `apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: skill-duplicate-runtime + namespace: ` + skillTestNamespace + ` +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: skill-agent-target + skills: + - name: my-skill + image: registry.k8s.io/pause:3.9 + mountPath: /agent/skills/my-skill + - name: my-skill + image: registry.k8s.io/pause:3.10 + mountPath: /agent/skills/my-skill-2 +` +} + +func skillDuplicateMountPathAgentRuntimeFixture() string { + return `apiVersion: agent.kagenti.dev/v1alpha1 +kind: AgentRuntime +metadata: + name: skill-dup-mount-runtime + namespace: ` + skillTestNamespace + ` +spec: + type: agent + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: skill-agent-target + skills: + - name: skill-a + image: registry.k8s.io/pause:3.9 + mountPath: /agent/skills/shared + - name: skill-b + image: registry.k8s.io/pause:3.10 + mountPath: /agent/skills/shared +` +} diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index 8396f3a6..e7ef90e0 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -477,13 +477,12 @@ func RestoreControllerArgs(namespace, deploy string, origArgs []string) error { // DeployController installs CRDs and deploys the controller-manager. func DeployController(namespace, img string) error { By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - if _, err := Run(cmd); err != nil && !strings.Contains(err.Error(), "already exists") { - return err + if err := ensureNamespaceReady(namespace, 60*time.Second); err != nil { + return fmt.Errorf("namespace %s not ready: %w", namespace, err) } By("labeling the namespace to enforce the restricted security policy") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + cmd := exec.Command("kubectl", "label", "--overwrite", "ns", namespace, "pod-security.kubernetes.io/enforce=restricted") if _, err := Run(cmd); err != nil { return err @@ -506,6 +505,31 @@ func DeployController(namespace, img string) error { return err } +// ensureNamespaceReady creates the namespace, waiting for any Terminating +// instance to be fully deleted first. A previous test's cleanup may have +// triggered namespace deletion that hasn't finished yet. +func ensureNamespaceReady(namespace string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + cmd := exec.Command("kubectl", "get", "ns", namespace, "-o", "jsonpath={.status.phase}") + output, err := Run(cmd) + if err != nil { + // Namespace doesn't exist — create it + createCmd := exec.Command("kubectl", "create", "ns", namespace) + _, createErr := Run(createCmd) + return createErr + } + phase := strings.TrimSpace(output) + if phase == "Active" { + return nil + } + // Namespace is Terminating — wait for it to be fully deleted + By(fmt.Sprintf("waiting for namespace %s to finish terminating (phase=%s)", namespace, phase)) + time.Sleep(2 * time.Second) + } + return fmt.Errorf("timed out waiting for namespace %s to finish terminating", namespace) +} + // EnsureCertManagerWebhookReady waits for the cert-manager webhook to be responsive. // This is needed when re-deploying after an undeploy, because the webhook's TLS // serving certificate can become stale. @@ -560,6 +584,74 @@ spec: return err == nil } +// EnableSkillImageVolumes creates a feature-gates ConfigMap with +// skillImageVolumes enabled and patches the controller Deployment to mount it. +func EnableSkillImageVolumes(namespace, deploy string) error { + By("creating feature-gates ConfigMap with skillImageVolumes enabled") + cmd := exec.Command("kubectl", "apply", "-f", "-", "-n", namespace) + cmd.Stdin = strings.NewReader(`apiVersion: v1 +kind: ConfigMap +metadata: + name: kagenti-feature-gates +data: + feature-gates.yaml: | + globalEnabled: true + envoyProxy: true + spiffeHelper: true + clientRegistration: true + injectTools: false + perWorkloadConfigResolution: false + combinedSidecar: false + skillImageVolumes: true +`) + if _, err := Run(cmd); err != nil { + return fmt.Errorf("failed to create feature-gates ConfigMap: %w", err) + } + + By("patching controller to mount feature-gates ConfigMap") + patch := `{"spec":{"template":{"spec":{` + + `"containers":[{"name":"manager",` + + `"volumeMounts":[{"name":"feature-gates","mountPath":"/etc/kagenti/feature-gates","readOnly":true}]}],` + + `"volumes":[{"name":"feature-gates","configMap":{"name":"kagenti-feature-gates"}}]}}}}` + cmd = exec.Command("kubectl", "patch", "deployment", deploy, + "-n", namespace, + "--type=strategic", + fmt.Sprintf("-p=%s", patch), + ) + if _, err := Run(cmd); err != nil { + return fmt.Errorf("failed to patch controller with feature-gates volume: %w", err) + } + + By("waiting for controller rollout after feature-gates patch") + return WaitForRollout(deploy, namespace, 2*time.Minute) +} + +// DisableSkillImageVolumes updates the feature-gates ConfigMap to disable +// skillImageVolumes. The feature gate loader's file watcher picks up changes. +func DisableSkillImageVolumes(namespace string) error { + By("updating feature-gates ConfigMap to disable skillImageVolumes") + cmd := exec.Command("kubectl", "apply", "-f", "-", "-n", namespace) + cmd.Stdin = strings.NewReader(`apiVersion: v1 +kind: ConfigMap +metadata: + name: kagenti-feature-gates +data: + feature-gates.yaml: | + globalEnabled: true + envoyProxy: true + spiffeHelper: true + clientRegistration: true + injectTools: false + perWorkloadConfigResolution: false + combinedSidecar: false + skillImageVolumes: false +`) + if _, err := Run(cmd); err != nil { + return fmt.Errorf("failed to update feature-gates ConfigMap: %w", err) + } + return nil +} + // UndeployController undeploys the controller-manager and uninstalls CRDs. func UndeployController() { By("undeploying the controller-manager") From 613566cbda8c07fd02179f7efbd9bed533cde813 Mon Sep 17 00:00:00 2001 From: Kevin Cogan Date: Mon, 25 May 2026 17:13:22 +0100 Subject: [PATCH 2/2] fix: address review feedback for OCI skill image mounting - Add mountPath denylist rejecting /proc, /sys, /dev, /var/run/secrets and path traversal (..) segments - Handle json.Marshal error instead of silently discarding - Remove unrelated combinedSidecar feature gate (out of scope) - Add unit tests for denylist validation Signed-off-by: Kevin Cogan --- charts/kagenti-operator/values.yaml | 4 --- .../controller/agentruntime_controller.go | 5 +++- .../webhook/config/feature_gate_loader.go | 1 - .../internal/webhook/config/feature_gates.go | 3 -- .../webhook/v1alpha1/agentruntime_webhook.go | 12 ++++++++ .../v1alpha1/agentruntime_webhook_test.go | 28 +++++++++++++++++++ kagenti-operator/test/utils/utils.go | 2 -- 7 files changed, 44 insertions(+), 11 deletions(-) diff --git a/charts/kagenti-operator/values.yaml b/charts/kagenti-operator/values.yaml index 171f8eae..d79eca4a 100644 --- a/charts/kagenti-operator/values.yaml +++ b/charts/kagenti-operator/values.yaml @@ -151,10 +151,6 @@ featureGates: # Default false — cached mode is faster and sufficient when namespace ConfigMaps # rarely change. Cache is cleared on webhook pod restart. perWorkloadConfigResolution: false - # combinedSidecar controls whether injection uses a single combined authbridge - # container instead of separate envoy-proxy + spiffe-helper + client-registration. - # Requires the authbridge image. Default false — separate sidecars. - combinedSidecar: false # skillImageVolumes controls whether AgentRuntime can mount OCI skill images # as Kubernetes ImageVolumes into agent pods. Requires Kubernetes 1.31+ with # the ImageVolume feature gate enabled. Default false. diff --git a/kagenti-operator/internal/controller/agentruntime_controller.go b/kagenti-operator/internal/controller/agentruntime_controller.go index 7352aae0..5fd27bf6 100644 --- a/kagenti-operator/internal/controller/agentruntime_controller.go +++ b/kagenti-operator/internal/controller/agentruntime_controller.go @@ -354,7 +354,10 @@ func (r *AgentRuntimeReconciler) applyWorkloadConfig(ctx context.Context, rt *ag for _, s := range rt.Spec.Skills { names = append(names, s.Name) } - b, _ := json.Marshal(names) + b, err := json.Marshal(names) + if err != nil { + logger.Error(err, "failed to marshal skill names") + } workloadAnnotations[AnnotationSkills] = string(b) } else { delete(workloadAnnotations, AnnotationSkills) diff --git a/kagenti-operator/internal/webhook/config/feature_gate_loader.go b/kagenti-operator/internal/webhook/config/feature_gate_loader.go index c0d3eb3f..2fac0177 100644 --- a/kagenti-operator/internal/webhook/config/feature_gate_loader.go +++ b/kagenti-operator/internal/webhook/config/feature_gate_loader.go @@ -180,7 +180,6 @@ func logFeatureGates(fg *FeatureGates, source string) { "envoyProxy", fg.EnvoyProxy, "injectTools", fg.InjectTools, "perWorkloadConfigResolution", fg.PerWorkloadConfigResolution, - "combinedSidecar", fg.CombinedSidecar, "skillImageVolumes", fg.SkillImageVolumes, ) log.Info("=============================================") diff --git a/kagenti-operator/internal/webhook/config/feature_gates.go b/kagenti-operator/internal/webhook/config/feature_gates.go index 48aec8ca..f3e61801 100644 --- a/kagenti-operator/internal/webhook/config/feature_gates.go +++ b/kagenti-operator/internal/webhook/config/feature_gates.go @@ -22,9 +22,6 @@ type FeatureGates struct { // true → resolved path: webhook reads namespace ConfigMaps at // admission time and injects literal env var values. PerWorkloadConfigResolution bool `json:"perWorkloadConfigResolution" yaml:"perWorkloadConfigResolution"` - // CombinedSidecar controls whether injection uses a single combined authbridge - // container instead of separate envoy-proxy + spiffe-helper + client-registration sidecars. - CombinedSidecar bool `json:"combinedSidecar" yaml:"combinedSidecar"` // SkillImageVolumes controls whether the AgentRuntime controller mounts OCI // skill images as Kubernetes ImageVolumes into agent pods. Requires // Kubernetes 1.31+ with the ImageVolume feature gate enabled. diff --git a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go index 8d097b9f..29be59e8 100644 --- a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go +++ b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook.go @@ -19,6 +19,7 @@ package v1alpha1 import ( "context" "fmt" + "strings" agentv1alpha1 "github.com/kagenti/operator/api/v1alpha1" ctrl "sigs.k8s.io/controller-runtime" @@ -141,6 +142,8 @@ func (v *AgentRuntimeValidator) checkDuplicateTargetRef(ctx context.Context, rt return nil } +var deniedMountPrefixes = []string{"/proc", "/sys", "/dev", "/var/run/secrets"} + func validateSkills(skills []agentv1alpha1.SkillImageRef) error { if len(skills) == 0 { return nil @@ -158,6 +161,15 @@ func validateSkills(skills []agentv1alpha1.SkillImageRef) error { return fmt.Errorf("spec.skills[%d]: duplicate mountPath %q", i, skill.MountPath) } seenPaths[skill.MountPath] = true + + if strings.Contains(skill.MountPath, "..") { + return fmt.Errorf("spec.skills[%d]: mountPath %q must not contain path traversal (..)", i, skill.MountPath) + } + for _, prefix := range deniedMountPrefixes { + if strings.HasPrefix(skill.MountPath, prefix) { + return fmt.Errorf("spec.skills[%d]: mountPath %q overlaps protected path %q", i, skill.MountPath, prefix) + } + } } return nil } diff --git a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go index 6831b473..d16e3a6d 100644 --- a/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go +++ b/kagenti-operator/internal/webhook/v1alpha1/agentruntime_webhook_test.go @@ -355,6 +355,34 @@ func TestValidateSkills(t *testing.T) { t.Errorf("unexpected error message: %v", err) } }) + + t.Run("path traversal rejected", func(t *testing.T) { + skills := []agentv1alpha1.SkillImageRef{ + {Name: "evil", Image: "img:v1", MountPath: "/agent/skills/../../etc/passwd"}, + } + err := validateSkills(skills) + if err == nil { + t.Fatal("expected error for path traversal") + } + if !strings.Contains(err.Error(), "path traversal") { + t.Errorf("unexpected error message: %v", err) + } + }) + + t.Run("protected path rejected", func(t *testing.T) { + for _, path := range []string{"/proc/self", "/sys/fs", "/dev/null", "/var/run/secrets/kubernetes.io/serviceaccount"} { + skills := []agentv1alpha1.SkillImageRef{ + {Name: "bad-mount", Image: "img:v1", MountPath: path}, + } + err := validateSkills(skills) + if err == nil { + t.Fatalf("expected error for protected path %q", path) + } + if !strings.Contains(err.Error(), "protected path") { + t.Errorf("unexpected error message for %q: %v", path, err) + } + } + }) } func TestValidateSkills_CreateIntegration(t *testing.T) { diff --git a/kagenti-operator/test/utils/utils.go b/kagenti-operator/test/utils/utils.go index e7ef90e0..2cc6b5a2 100644 --- a/kagenti-operator/test/utils/utils.go +++ b/kagenti-operator/test/utils/utils.go @@ -601,7 +601,6 @@ data: clientRegistration: true injectTools: false perWorkloadConfigResolution: false - combinedSidecar: false skillImageVolumes: true `) if _, err := Run(cmd); err != nil { @@ -643,7 +642,6 @@ data: clientRegistration: true injectTools: false perWorkloadConfigResolution: false - combinedSidecar: false skillImageVolumes: false `) if _, err := Run(cmd); err != nil {