From 55859d35aef8bd34cf0c45a80d056ec319821e3c Mon Sep 17 00:00:00 2001 From: Aniruddha Basak Date: Mon, 12 Feb 2024 14:38:05 +0530 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20Add=20timeout=20while=20evaluat?= =?UTF-8?q?ing=20CEL=20expression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add timeout while evaluating CEL expression Signed-off-by: Aniruddha Basak --- api/v1alpha1/clusteraddon_types.go | 12 + api/v1alpha1/conditions_const.go | 22 ++ .../clusterstack.x-k8s.io_clusteraddons.yaml | 15 + extension/config/default/sa-roles.yaml | 1 + extension/go.mod | 5 +- extension/go.sum | 2 + extension/handlers/hooks.go | 69 +++- .../api/v1alpha1/clusteraddon_types.go | 12 + .../api/v1alpha1/conditions_const.go | 22 ++ extension/vendor/modules.txt | 2 + .../controller/clusteraddon_controller.go | 319 ++++++++++++++++++ pkg/clusteraddon/clusteraddon.go | 83 +++++ .../docker/ferrol/1-27/clusteraddon.yaml | 25 ++ 13 files changed, 570 insertions(+), 19 deletions(-) create mode 100644 pkg/clusteraddon/clusteraddon.go create mode 100644 test/cluster-stacks/docker/ferrol/1-27/clusteraddon.yaml diff --git a/api/v1alpha1/clusteraddon_types.go b/api/v1alpha1/clusteraddon_types.go index 28c8380e9..672b20f3a 100644 --- a/api/v1alpha1/clusteraddon_types.go +++ b/api/v1alpha1/clusteraddon_types.go @@ -49,6 +49,18 @@ type ClusterAddonStatus struct { // +optional Resources []*Resource `json:"resources,omitempty"` + // CurrentHook specifies the current running Hook. + // +optional + CurrentHook string `json:"currentHook,omitempty"` + + // KubernetesVersion is the kubernetes version of the current cluster stack release. + // +optional + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + + // HelmChartStatus defines the status of helm chart in the cluster addon. + // +optional + HelmChartStatus map[string]HelmChartStatusConditions `json:"helmChartStatus,omitempty"` + // +optional // +kubebuilder:default:=false Ready bool `json:"ready"` diff --git a/api/v1alpha1/conditions_const.go b/api/v1alpha1/conditions_const.go index c65895015..afdb7589d 100644 --- a/api/v1alpha1/conditions_const.go +++ b/api/v1alpha1/conditions_const.go @@ -26,6 +26,28 @@ const ( ControlPlaneNotReadyReason = "ControlPlaneNotReady" ) +const ( + // EvaluatedCELCondition reports on whether the CEL expression is evaluated properly. + EvaluatedCELCondition clusterv1.ConditionType = "EvaluatedCEL" + + // FailedToEvaluatePreConditionReason is used when some pre CEL expression have been failed to evaluate. + FailedToEvaluatePreConditionReason = "FailedToEvaluatePreCondition" + + // FailedToEvaluatePostConditionReason is used when some post CEL expression have been failed to evaluate. + FailedToEvaluatePostConditionReason = "FailedToEvaluatePostCondition" + + // CELEvaluationTimeoutReason is used when CEL expression have been failed to evaluate and timeout(1 min) has occurred. + CELEvaluationTimeoutReason = "CELEvaluationTimeout" +) + +const ( + // HelmChartFoundCondition reports when mentioned helm chart is present in the cluster addon tar archive. + HelmChartFoundCondition = "HelmChartFound" + + // HelmChartMissingReason is used when mentioned helm chart is missing in the cluster addon tar archive. + HelmChartMissingReason = "HelmChartMissing" +) + const ( // HelmChartAppliedCondition reports on whether the relevant helm chart has been applied. HelmChartAppliedCondition clusterv1.ConditionType = "HelmChartApplied" diff --git a/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml b/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml index 5bbd04254..3f0f3028f 100644 --- a/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml +++ b/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml @@ -152,6 +152,21 @@ spec: - type type: object type: array + currentHook: + description: CurrentHook specifies the current running Hook. + type: string + helmChartStatus: + additionalProperties: + description: HelmChartStatusConditions defines the status of helm + chart in the cluster addon. + type: string + description: HelmChartStatus defines the status of helm chart in the + cluster addon. + type: object + kubernetesVersion: + description: KubernetesVersion is the kubernetes version of the current + cluster stack release. + type: string ready: default: false type: boolean diff --git a/extension/config/default/sa-roles.yaml b/extension/config/default/sa-roles.yaml index 8b8cd92bb..42aa444f7 100644 --- a/extension/config/default/sa-roles.yaml +++ b/extension/config/default/sa-roles.yaml @@ -18,6 +18,7 @@ rules: - clusterstack.x-k8s.io resources: - clusteraddons + - clusteraddons/status verbs: - get - patch diff --git a/extension/go.mod b/extension/go.mod index 5849c8b33..3a72b990c 100644 --- a/extension/go.mod +++ b/extension/go.mod @@ -1,6 +1,8 @@ module github.com/SovereignCloudStack/cluster-stack-operator/extension -go 1.19 +go 1.21 + +toolchain go1.21.4 replace github.com/SovereignCloudStack/cluster-stack-operator => ../ @@ -55,6 +57,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.14.0 // indirect golang.org/x/sys v0.14.0 // indirect diff --git a/extension/go.sum b/extension/go.sum index fb99b26a7..eda2993f8 100644 --- a/extension/go.sum +++ b/extension/go.sum @@ -181,6 +181,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= diff --git a/extension/handlers/hooks.go b/extension/handlers/hooks.go index 760580ef4..a3c3b0389 100644 --- a/extension/handlers/hooks.go +++ b/extension/handlers/hooks.go @@ -21,6 +21,7 @@ import ( "fmt" csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" "sigs.k8s.io/cluster-api/util/conditions" @@ -47,28 +48,43 @@ func (e *ExtensionHandler) DoBeforeClusterUpgrade(ctx context.Context, request * key := types.NamespacedName{Name: fmt.Sprintf("cluster-addon-%s", request.Cluster.GetName()), Namespace: request.Cluster.GetNamespace()} clusterAddon := &csov1alpha1.ClusterAddon{} if err := e.client.Get(ctx, key, clusterAddon); err != nil { + log.Error(err, "failed to get cluster addon") response.SetStatus(runtimehooksv1.ResponseStatusFailure) response.SetMessage(err.Error()) return } - patchHelper, err := patch.NewHelper(clusterAddon, e.client) - if err != nil { - response.SetStatus(runtimehooksv1.ResponseStatusFailure) - response.SetMessage(fmt.Errorf("failed to init patch helper: %w", err).Error()) - } + if clusterAddon.Spec.Hook != "BeforeClusterUpgrade" { + patchHelper, err := patch.NewHelper(clusterAddon, e.client) + if err != nil { + log.Error(err, "failed to create patch helper") + response.SetStatus(runtimehooksv1.ResponseStatusFailure) + response.SetMessage(err.Error()) + } + + clusterAddon.Spec.Hook = "BeforeClusterUpgrade" + conditions.Delete(clusterAddon, csov1alpha1.HelmChartAppliedCondition) - defer func() { if err := patchHelper.Patch(ctx, clusterAddon); err != nil { + log.Error(err, "failed to patch cluster addon") response.SetStatus(runtimehooksv1.ResponseStatusFailure) - response.SetMessage(fmt.Errorf("failed to patch clusterAddon: %w", err).Error()) + response.SetMessage(err.Error()) } - }() - if conditions.IsFalse(clusterAddon, csov1alpha1.HelmChartAppliedCondition) { - // wait N seconds response.SetRetryAfterSeconds(10) + return + } + + if e.isTimeoutConditionPresent(ctx, key, clusterAddon) { + message := fmt.Sprintf("timeout occurred while evaluating CEL expression: %s", conditions.GetMessage(clusterAddon, csov1alpha1.EvaluatedCELCondition)) + log.Info(message) response.SetStatus(runtimehooksv1.ResponseStatusFailure) + response.SetMessage(message) + return + } + + if !utils.IsPresentAndTrue(ctx, e.client, key, clusterAddon, csov1alpha1.HelmChartAppliedCondition) { + response.SetRetryAfterSeconds(5) return } @@ -97,19 +113,36 @@ func (e *ExtensionHandler) DoAfterControlPlaneInitialized(ctx context.Context, r return } - patchHelper, err := patch.NewHelper(clusterAddon, e.client) - if err != nil { - response.SetStatus(runtimehooksv1.ResponseStatusFailure) - response.SetMessage(fmt.Errorf("failed to init patch helper: %w", err).Error()) - } + if clusterAddon.Spec.Hook != "AfterControlPlaneInitialized" { + patchHelper, err := patch.NewHelper(clusterAddon, e.client) + if err != nil { + log.Error(err, "failed to create patch helper") + response.SetStatus(runtimehooksv1.ResponseStatusFailure) + response.SetMessage(fmt.Errorf("failed to init patch helper: %w", err).Error()) + } - defer func() { - if err := patchHelper.Patch(ctx, clusterAddon); err != nil { + clusterAddon.Spec.Hook = "AfterControlPlaneInitialized" + conditions.Delete(clusterAddon, csov1alpha1.HelmChartAppliedCondition) + + if err := patchHelper.Patch(ctx, clusterAddon, patch.WithForceOverwriteConditions{}); err != nil { + log.Error(err, "failed to patch cluster addon") response.SetStatus(runtimehooksv1.ResponseStatusFailure) response.SetMessage(fmt.Errorf("failed to patch clusterAddon: %w", err).Error()) } - }() + } + + if e.isTimeoutConditionPresent(ctx, key, clusterAddon) { + message := fmt.Sprintf("timeout occurred while evaluating CEL expression: %s", conditions.GetMessage(clusterAddon, csov1alpha1.EvaluatedCELCondition)) + log.Info(message) + response.SetStatus(runtimehooksv1.ResponseStatusFailure) + response.SetMessage(message) + return + } response.SetStatus(runtimehooksv1.ResponseStatusSuccess) return } + +func (e *ExtensionHandler) isTimeoutConditionPresent(ctx context.Context, key types.NamespacedName, getter conditions.Getter) bool { + return utils.IsPresentAndFalseWithReason(ctx, e.client, key, getter, csov1alpha1.EvaluatedCELCondition, csov1alpha1.CELEvaluationTimeoutReason) +} diff --git a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusteraddon_types.go b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusteraddon_types.go index 28c8380e9..672b20f3a 100644 --- a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusteraddon_types.go +++ b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusteraddon_types.go @@ -49,6 +49,18 @@ type ClusterAddonStatus struct { // +optional Resources []*Resource `json:"resources,omitempty"` + // CurrentHook specifies the current running Hook. + // +optional + CurrentHook string `json:"currentHook,omitempty"` + + // KubernetesVersion is the kubernetes version of the current cluster stack release. + // +optional + KubernetesVersion string `json:"kubernetesVersion,omitempty"` + + // HelmChartStatus defines the status of helm chart in the cluster addon. + // +optional + HelmChartStatus map[string]HelmChartStatusConditions `json:"helmChartStatus,omitempty"` + // +optional // +kubebuilder:default:=false Ready bool `json:"ready"` diff --git a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/conditions_const.go b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/conditions_const.go index c65895015..afdb7589d 100644 --- a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/conditions_const.go +++ b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/conditions_const.go @@ -26,6 +26,28 @@ const ( ControlPlaneNotReadyReason = "ControlPlaneNotReady" ) +const ( + // EvaluatedCELCondition reports on whether the CEL expression is evaluated properly. + EvaluatedCELCondition clusterv1.ConditionType = "EvaluatedCEL" + + // FailedToEvaluatePreConditionReason is used when some pre CEL expression have been failed to evaluate. + FailedToEvaluatePreConditionReason = "FailedToEvaluatePreCondition" + + // FailedToEvaluatePostConditionReason is used when some post CEL expression have been failed to evaluate. + FailedToEvaluatePostConditionReason = "FailedToEvaluatePostCondition" + + // CELEvaluationTimeoutReason is used when CEL expression have been failed to evaluate and timeout(1 min) has occurred. + CELEvaluationTimeoutReason = "CELEvaluationTimeout" +) + +const ( + // HelmChartFoundCondition reports when mentioned helm chart is present in the cluster addon tar archive. + HelmChartFoundCondition = "HelmChartFound" + + // HelmChartMissingReason is used when mentioned helm chart is missing in the cluster addon tar archive. + HelmChartMissingReason = "HelmChartMissing" +) + const ( // HelmChartAppliedCondition reports on whether the relevant helm chart has been applied. HelmChartAppliedCondition clusterv1.ConditionType = "HelmChartApplied" diff --git a/extension/vendor/modules.txt b/extension/vendor/modules.txt index a90f24ece..ea74b4542 100644 --- a/extension/vendor/modules.txt +++ b/extension/vendor/modules.txt @@ -166,6 +166,8 @@ go.uber.org/zap/zapcore # golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ## explicit; go 1.20 golang.org/x/exp/maps +# golang.org/x/mod v0.14.0 +## explicit; go 1.18 # golang.org/x/net v0.18.0 ## explicit; go 1.18 golang.org/x/net/context diff --git a/internal/controller/clusteraddon_controller.go b/internal/controller/clusteraddon_controller.go index 7e13a7b34..7c3e8444c 100644 --- a/internal/controller/clusteraddon_controller.go +++ b/internal/controller/clusteraddon_controller.go @@ -69,6 +69,7 @@ type ClusterAddonReconciler struct { KubeClientFactory kube.Factory WatchFilterValue string WorkloadClusterFactory workloadcluster.Factory + addonConditionTimeout time.Time } //+kubebuilder:rbac:groups=clusterstack.x-k8s.io,resources=clusteraddons,verbs=get;list;watch;create;update;patch;delete @@ -227,6 +228,65 @@ func (r *ClusterAddonReconciler) Reconcile(ctx context.Context, req reconcile.Re clusterAddon.Spec.ClusterStack = cluster.Spec.Topology.Class clusterAddon.Status.Ready = true return ctrl.Result{}, nil + + } else { + if clusterAddon.Spec.Hook != clusterAddon.Status.CurrentHook { + clusterAddon.Status.HelmChartStatus = make(map[string]csov1alpha1.HelmChartStatusConditions) + clusterAddon.Status.CurrentHook = clusterAddon.Spec.Hook + clusterAddon.Status.Ready = false + } + + if clusterAddon.Status.Ready { + return reconcile.Result{}, nil + } + + // return if timeout occurs. + if conditions.GetReason(clusterAddon, csov1alpha1.EvaluatedCELCondition) == csov1alpha1.CELEvaluationTimeoutReason { + return reconcile.Result{}, nil + } + + // metadata := releaseAsset.Meta + + // only apply the Helm chart again if the Helm chart version has also changed from one cluster stack release to the other + // if clusterAddon.Spec.Version == metadata.Versions.Components.ClusterAddon { + // clusterAddon.Status.Ready = true + // return reconcile.Result{}, nil + // } + + in.addonStagesInput, err = getAddonStagesInput(r, in.restConfig, in.clusterAddonChartPath) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to get addon stages input: %w", err) + } + + // clusteraddon.yaml in the release. + clusterAddonConfig, err := clusteraddon.ParseClusterAddonConfig(in.clusterAddonConfigPath) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to parse clusteraddon.yaml config: %w", err) + } + + for _, stage := range clusterAddonConfig.AddonStages[in.clusterAddon.Spec.Hook] { + shouldRequeue, err := r.executeStage(ctx, stage, in) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to execute stage: %w", err) + } + if shouldRequeue { + return reconcile.Result{RequeueAfter: 20 * time.Second}, nil + } + } + + // return if timeout occurs. + if conditions.GetReason(clusterAddon, csov1alpha1.EvaluatedCELCondition) == csov1alpha1.CELEvaluationTimeoutReason { + return reconcile.Result{}, nil + } + + // Helm chart has been applied successfully + // clusterAddon.Spec.Version = metadata.Versions.Components.ClusterAddon + conditions.MarkTrue(clusterAddon, csov1alpha1.HelmChartAppliedCondition) + + // store the release kubernetes version and current hook + clusterAddon.Status.KubernetesVersion = releaseAsset.Meta.Versions.Kubernetes + clusterAddon.Status.CurrentHook = clusterAddon.Spec.Hook + clusterAddon.Status.Ready = true } // if condition is false we have not yet successfully applied the helm chart @@ -291,6 +351,265 @@ func (r *ClusterAddonReconciler) templateAndApplyClusterAddonHelmChart(ctx conte return shouldRequeue, nil } +func (r *ClusterAddonReconciler) executeStage(ctx context.Context, stage clusteraddon.Stage, in templateAndApplyClusterAddonInput) (bool, error) { + + var ( + newResources []*csov1alpha1.Resource + shouldRequeue bool + buildTemplate []byte + err error + ) + + _, exists := in.chartMap[stage.HelmChartName] + if !exists { + // do not reconcile by returning error, just create an event. + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.HelmChartFoundCondition, + csov1alpha1.HelmChartMissingReason, + clusterv1.ConditionSeverityInfo, + "helm chart name doesn't exists in the cluster addon helm chart: %q", + stage.HelmChartName, + ) + return false, nil + } + +check: + switch in.clusterAddon.Status.HelmChartStatus[stage.HelmChartName] { + case csov1alpha1.None: + // If no WaitForPreCondition is mentioned. + if !reflect.DeepEqual(stage.WaitForPreCondition, clusteraddon.WaitForCondition{}) { + // check if timeout is already set or not. + if r.addonConditionTimeout.IsZero() { + r.addonConditionTimeout = time.Now() + } + + // Evaluate the condition. + if err := getDynamicResourceAndEvaluateCEL(ctx, in.dynamicClient, in.discoverClient, stage.WaitForPreCondition); err != nil { + if errors.Is(err, clusteraddon.ConditionNotMatchError) { + // check if timeout has elapsed. + if time.Since(r.addonConditionTimeout) > stage.WaitForPreCondition.Timeout { + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.EvaluatedCELCondition, + csov1alpha1.CELEvaluationTimeoutReason, + clusterv1.ConditionSeverityError, + "timeout occurred evaluating pre condition for stage: %q: %s", stage.HelmChartName, err.Error(), + ) + + return false, nil + } + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.EvaluatedCELCondition, + csov1alpha1.FailedToEvaluatePreConditionReason, + clusterv1.ConditionSeverityInfo, + "failed to successfully evaluate pre condition: %q: %s", stage.HelmChartName, err.Error(), + ) + + in.clusterAddon.Status.HelmChartStatus[stage.HelmChartName] = csov1alpha1.WaitingForPreCondition + + return true, nil + } + return false, fmt.Errorf("failed to get dynamic resource and evaluate cel expression for pre condition: %w", err) + } + + // reset the timeout + r.addonConditionTimeout = time.Time{} + } + in.clusterAddon.Status.HelmChartStatus[stage.HelmChartName] = csov1alpha1.ApplyingOrDeleting + goto check + + case csov1alpha1.ApplyingOrDeleting: + if stage.Action == clusteraddon.Apply { + newResources, shouldRequeue, err = helmTemplateAndApply(ctx, in.kubeClient, in, stage.HelmChartName, buildTemplate, false) + if err != nil { + return false, fmt.Errorf("failed to helm template and apply: %w", err) + } + if shouldRequeue { + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.HelmChartAppliedCondition, + csov1alpha1.FailedToApplyObjectsReason, + clusterv1.ConditionSeverityInfo, + "failed to successfully apply helm chart: %q", stage.HelmChartName, + ) + + return true, nil + } + in.clusterAddon.Status.HelmChartStatus[stage.HelmChartName] = csov1alpha1.WaitingForPostCondition + goto check + + } else { + // Delete part + newResources, shouldRequeue, err = helmTemplateAndDelete(ctx, in, stage.HelmChartName, buildTemplate) + if err != nil { + return false, fmt.Errorf("failed to delete helm chart: %w", err) + } + if shouldRequeue { + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.HelmChartDeletedCondition, + csov1alpha1.FailedToDeleteObjectsReason, + clusterv1.ConditionSeverityInfo, + "failed to successfully delete helm chart: %q", stage.HelmChartName, + ) + + return true, nil + } + in.clusterAddon.Status.HelmChartStatus[stage.HelmChartName] = csov1alpha1.WaitingForPostCondition + goto check + } + + case csov1alpha1.WaitingForPostCondition: + // If no WaitForPostCondition is mentioned. + if !reflect.DeepEqual(stage.WaitForPostCondition, clusteraddon.WaitForCondition{}) { + // check if timeout is already set or not. + if r.addonConditionTimeout.IsZero() { + r.addonConditionTimeout = time.Now() + } + + // Evaluate the condition. + if err := getDynamicResourceAndEvaluateCEL(ctx, in.dynamicClient, in.discoverClient, stage.WaitForPostCondition); err != nil { + if errors.Is(err, clusteraddon.ConditionNotMatchError) { + // check if timeout has elapsed. + if time.Since(r.addonConditionTimeout) > stage.WaitForPostCondition.Timeout { + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.EvaluatedCELCondition, + csov1alpha1.CELEvaluationTimeoutReason, + clusterv1.ConditionSeverityError, + "timeout occurred evaluating post condition for stage: %q: %s", stage.HelmChartName, err.Error(), + ) + + return false, nil + } + + conditions.MarkFalse( + in.clusterAddon, + csov1alpha1.EvaluatedCELCondition, + csov1alpha1.FailedToEvaluatePostConditionReason, + clusterv1.ConditionSeverityInfo, + "failed to successfully evaluate post condition: %q: %s", stage.HelmChartName, err.Error(), + ) + + return true, nil + } + return false, fmt.Errorf("failed to get dynamic resource and evaluate cel expression for pre condition: %w", err) + } + + // reset the timeout + r.addonConditionTimeout = time.Time{} + } + in.clusterAddon.Status.HelmChartStatus[stage.HelmChartName] = csov1alpha1.Done + } + + in.clusterAddon.Status.Resources = append(in.clusterAddon.Status.Resources, newResources...) + + return false, nil +} + +func helmTemplateAndApply(ctx context.Context, kubeClient kube.Client, in templateAndApplyClusterAddonInput, helmChartName string, buildTemplate []byte, shouldDelete bool) ([]*csov1alpha1.Resource, bool, error) { + subDirPath := filepath.Join(in.destinationClusterAddonChartDir, helmChartName) + + helmTemplate, err := helmTemplateClusterAddon(subDirPath, buildTemplate, true) + if err != nil { + return nil, false, fmt.Errorf("failed to template helm chart: %w", err) + } + + newResources, shouldRequeue, err := kubeClient.Apply(ctx, helmTemplate, in.clusterAddon.Status.Resources, shouldDelete) + if err != nil { + return nil, false, fmt.Errorf("failed to apply objects from cluster addon Helm chart: %w", err) + } + + return newResources, shouldRequeue, nil +} + +func helmTemplateAndDelete(ctx context.Context, in templateAndApplyClusterAddonInput, helmChartName string, buildTemplate []byte) ([]*csov1alpha1.Resource, bool, error) { + subDirPath := filepath.Join(in.destinationClusterAddonChartDir, helmChartName) + + helmTemplate, err := helmTemplateClusterAddon(subDirPath, buildTemplate, true) + if err != nil { + return nil, false, fmt.Errorf("failed to template helm chart: %w", err) + } + + newResources, shouldRequeue, err := in.kubeClient.Delete(ctx, helmTemplate, in.clusterAddon.Status.Resources) + if err != nil { + return nil, false, fmt.Errorf("failed to delete objects from cluster addon Helm chart: %w", err) + } + + return newResources, shouldRequeue, nil +} + +func getDynamicResourceAndEvaluateCEL(ctx context.Context, dynamicClient *dynamic.DynamicClient, discoveryClient *discovery.DiscoveryClient, waitCondition clusteraddon.WaitForCondition) error { + var evalMap = make(map[string]interface{}) + + env, err := cel.NewEnv() + if err != nil { + return fmt.Errorf("environment creation error: %w", err) + } + + for _, object := range waitCondition.Objects { + env, err = env.Extend( + cel.Variable(object.Key, celtypes.NewMapType(cel.StringType, cel.AnyType)), + ) + if err != nil { + return fmt.Errorf("failed to extend the current cel environment with %q: %w", object.Key, err) + } + } + + ast, iss := env.Compile(waitCondition.Conditions) + if !errors.Is(iss.Err(), nil) { + return fmt.Errorf("failed to compile cel expression: %q: %w", waitCondition.Conditions, err) + } + + prg, err := env.Program(ast) + if err != nil { + return fmt.Errorf("failed to generate an evaluable instance of the Ast within the environment: %w", err) + } + + for _, object := range waitCondition.Objects { + splittedAPIVersion := strings.Split(object.APIVersion, "/") + + gvr := schema.GroupVersionKind{ + Kind: object.Kind, + } + if len(splittedAPIVersion) == 2 { + gvr.Group = splittedAPIVersion[0] + gvr.Version = splittedAPIVersion[1] + } else { + gvr.Group = "" + gvr.Version = splittedAPIVersion[0] + } + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(discoveryClient)) + mapping, err := mapper.RESTMapping(schema.GroupKind{Group: gvr.Group, Kind: gvr.Kind}, gvr.Version) + if err != nil { + return fmt.Errorf("error creating rest mapping: %w", err) + } + + resource := dynamicClient.Resource(mapping.Resource) + + unstructuredObj, err := resource.Namespace(object.Namespace).Get(ctx, object.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get object(name: %s, namespace: %s) from dynamic client: %w", object.Name, object.Namespace, err) + } + + evalMap[object.Key] = unstructuredObj.Object + } + + out, _, err := prg.Eval(evalMap) + if err != nil { + return fmt.Errorf("failed to evaluate the Ast and environment against the input vars: %w", err) + } + + if out.Value() != true { + return fmt.Errorf("failed to evaluate the cel expression, please check again: %w", clusteraddon.ConditionNotMatchError) + } + + return nil +} + func buildTemplateFromClusterAddonValues(ctx context.Context, addonValuePath string, cluster *clusterv1.Cluster, c client.Client) ([]byte, error) { data, err := os.ReadFile(filepath.Clean(addonValuePath)) if err != nil { diff --git a/pkg/clusteraddon/clusteraddon.go b/pkg/clusteraddon/clusteraddon.go new file mode 100644 index 000000000..b36144ead --- /dev/null +++ b/pkg/clusteraddon/clusteraddon.go @@ -0,0 +1,83 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cluster addon contains function for cluster addon config operations. +package clusteraddon + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "gopkg.in/yaml.v3" +) + +// Action is the type for helm chart action (e.g. - apply, delete). +type Action string + +// ConditionNotMatchError is used when the specified CEL expression doesn't match in the stage. +var ConditionNotMatchError = errors.New("condition don't match") + +var ( + // Apply applies a helm chart. + Apply = Action("apply") + + // Delete deletes a helm chart. + Delete = Action("delete") +) + +type Object struct { + Key string `yaml:"key"` + APIVersion string `yaml:"apiVersion"` + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Namespace string `yaml:"namespace"` +} + +type WaitForCondition struct { + Timeout time.Duration `yaml:"timeout"` + Objects []Object `yaml:"objects"` + Conditions string `yaml:"conditions"` +} + +type Stage struct { + HelmChartName string `yaml:"helmChartName"` + Action Action `yaml:"action"` + WaitForPreCondition WaitForCondition `yaml:"waitForPreCondition,omitempty"` + WaitForPostCondition WaitForCondition `yaml:"waitForPostCondition,omitempty"` +} + +type ClusterAddonConfig struct { + APIVersion string `yaml:"apiVersion"` + ClusterAddonVersion string `yaml:"clusterAddonVersion"` + AddonStages map[string][]Stage `yaml:"addonStages"` +} + +func ParseClusterAddonConfig(path string) (ClusterAddonConfig, error) { + data, err := os.ReadFile(filepath.Clean(path)) + if err != nil { + return ClusterAddonConfig{}, fmt.Errorf("failed to read file: %q: %w", path, err) + } + + var clusterAddon ClusterAddonConfig + if err := yaml.Unmarshal(data, &clusterAddon); err != nil { + return ClusterAddonConfig{}, fmt.Errorf("failed to parse cluster addon yaml: %w", err) + } + + return clusterAddon, nil +} diff --git a/test/cluster-stacks/docker/ferrol/1-27/clusteraddon.yaml b/test/cluster-stacks/docker/ferrol/1-27/clusteraddon.yaml new file mode 100644 index 000000000..9857eb9a4 --- /dev/null +++ b/test/cluster-stacks/docker/ferrol/1-27/clusteraddon.yaml @@ -0,0 +1,25 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - helmChartName: cni + action: apply + - helmChartName: metrics-server + action: apply + BeforeClusterUpgrade: + - helmChartName: ngnix + action: apply + waitForPostCondition: + objects: + - key: msdeployment + apiVersion: apps/v1 + kind: Deployment + name: metrics-server + namespace: kube-system + - key: ngnix + apiVersion: apps/v1 + kind: Deployment + name: cluster-addon-ngnix + namespace: kube-system + conditions: msdeployment.spec.replicas == ngnix.spec.replicas + timeout: 1m From 72cedc4f7a818d1be49592021f9c5ae9fc79601d Mon Sep 17 00:00:00 2001 From: Aniruddha Basak Date: Thu, 15 Feb 2024 12:31:20 +0530 Subject: [PATCH 2/2] =?UTF-8?q?fixup!=20=E2=9C=A8=20Add=20timeout=20while?= =?UTF-8?q?=20evaluating=20CEL=20expression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../clusterstack.x-k8s.io_clusteraddons.yaml | 5 +- .../api/v1alpha1/clusterstack_types.go | 2 +- .../pkg/version/channel.go | 12 +- .../pkg/version/version.go | 128 ++++-------------- go.mod | 2 +- 5 files changed, 35 insertions(+), 114 deletions(-) diff --git a/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml b/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml index 3f0f3028f..4d16a9cfa 100644 --- a/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml +++ b/config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml @@ -156,10 +156,7 @@ spec: description: CurrentHook specifies the current running Hook. type: string helmChartStatus: - additionalProperties: - description: HelmChartStatusConditions defines the status of helm - chart in the cluster addon. - type: string + additionalProperties: {} description: HelmChartStatus defines the status of helm chart in the cluster addon. type: object diff --git a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusterstack_types.go b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusterstack_types.go index e36c3cb5a..2cad61a94 100644 --- a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusterstack_types.go +++ b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1/clusterstack_types.go @@ -39,7 +39,7 @@ type ClusterStackSpec struct { // Channel specifies the release channel of the cluster stack. Defaults to 'stable'. // +kubebuilder:default:=stable - // +kubebuilder:validation:Enum=stable;custom + // +kubebuilder:validation:enum=stable;alpha;beta;rc Channel version.Channel `json:"channel,omitempty"` // Versions is a list of version of the cluster stack that should be available in the management cluster. diff --git a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/channel.go b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/channel.go index 1dd33c6fa..3acc415cd 100644 --- a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/channel.go +++ b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/channel.go @@ -23,12 +23,18 @@ type Channel string const ( // ChannelStable is the stable channel. ChannelStable = Channel("stable") - // ChannelCustom is the custom channel where git hash is used in the versioning. - ChannelCustom = Channel("custom") + // ChannelAlpha is the alpha channel. + ChannelAlpha = Channel("alpha") + // ChannelBeta is the beta channel. + ChannelBeta = Channel("beta") + // ChannelRC is the rc channel. + ChannelRC = Channel("rc") ) // IsValid returns true if the release channel is valid. func (c Channel) IsValid() bool { return c == ChannelStable || - c == ChannelCustom + c == ChannelAlpha || + c == ChannelBeta || + c == ChannelRC } diff --git a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/version.go b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/version.go index ec77f7cf8..69175d105 100644 --- a/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/version.go +++ b/extension/vendor/github.com/SovereignCloudStack/cluster-stack-operator/pkg/version/version.go @@ -34,28 +34,27 @@ import ( type Version struct { Major int Channel Channel - Patch string + Patch int } -// ParseVersionString returns a Version struct from a version string like - -// "v1", "v1-alpha-1", "v1-beta-3", etc. -func ParseVersionString(version string) (Version, error) { - var ( - major int - patch string - err error - ) +// New returns a Version struct from a version string +// Sample allowed inputs: "v1-alpha-1", "v1", "v1-alpha-0" +// Sample disallowed inputs: "v1-alpha", "v1-alpha-1.0", "v1-alpha-1.0.0", "v1-alpha.", "v1.0-alpha.1". +func New(version string) (Version, error) { + var major, patch int + var err error channel := ChannelStable - re := regexp.MustCompile(`^v\d+(-\b\w+\b\-\w+)?$`) - match := re.MatchString(version) - if !match { + re := regexp.MustCompile(`^v\d+(-\b\w+\b-\d+)?$`) + match := re.FindStringSubmatch(version) + + if len(match) == 0 { return Version{}, fmt.Errorf("invalid version string %s", version) } // match[0] is the entire string e.g "v1-alpha-1" or "v1" // split match[0] with "-" as the delimiter - ver := strings.Split(version, "-") + ver := strings.Split(match[0], "-") // ver[0] is the major version // trim the "v" prefix and then convert to int @@ -68,57 +67,9 @@ func ParseVersionString(version string) (Version, error) { // ver[2] is the patch if len(ver) == 3 { channel = Channel(ver[1]) - patch = ver[2] - } - clusterStackVersion := Version{ - Major: major, - Channel: channel, - Patch: patch, - } - if err := clusterStackVersion.Validate(); err != nil { - return Version{}, err - } - - return clusterStackVersion, nil -} - -// New returns a Version struct from a version string -// Sample allowed inputs: "v1-alpha.1", "v1", "v1-alpha.0" -// Sample disallowed inputs: "v1-alpha", "v1-alpha-1.0", "v1-alpha-1.0.0", "v1-alpha.", "v1.0-alpha.1". -func New(version string) (Version, error) { - var ( - major int - patch string - err error - ) - channel := ChannelStable - - re := regexp.MustCompile(`^v\d+(-\b\w+\b\.\w+)?$`) - match := re.MatchString(version) - if !match { - return Version{}, fmt.Errorf("invalid version string %s", version) - } - - // match[0] is the entire string e.g "v1-alpha.1" or "v1" - // split match[0] with "-" as the delimiter - ver := strings.Split(version, "-") - - // ver[0] is the major version - // trim the "v" prefix and then convert to int - if major, err = strconv.Atoi(strings.TrimPrefix(ver[0], "v")); err != nil { - return Version{}, fmt.Errorf("invalid major version %s", ver[0]) - } - - // If the length of ver is 2, then the version string is of the form "v1-alpha.1", and split it - - // ver[0] is the channel - // ver[1] is the patch - if len(ver) == 2 { - splittedChannelPatch := strings.Split(ver[1], ".") - if len(splittedChannelPatch) != 2 { - return Version{}, fmt.Errorf("invalid version string %s", version) + if patch, err = strconv.Atoi(ver[2]); err != nil { + return Version{}, fmt.Errorf("invalid patch value in version %s", ver[2]) } - channel = Channel(splittedChannelPatch[0]) - patch = splittedChannelPatch[1] } clusterStackVersion := Version{ @@ -135,15 +86,13 @@ func New(version string) (Version, error) { // FromReleaseTag returns a Version struct from a release tag string. func FromReleaseTag(releaseTag string) (Version, error) { v := strings.Split(releaseTag, "-") - if len(v) != 5 && len(v) != 7 { + if len(v) != 5 && len(v) != 6 { return Version{}, fmt.Errorf("invalid release tag %s", releaseTag) } - // for docker-ferrol-1-26-v1 type tag, v[4] is the version if len(v) == 5 { - return ParseVersionString(v[4]) + return New(v[4]) } - // for docker-ferrol-1-26-v1-alpha-0 type tag, v[4] is the version and v[5] is the release channel + patch version - return ParseVersionString(fmt.Sprintf("%s-%s-%s", v[4], v[5], v[6])) + return New(fmt.Sprintf("%s-%s", v[4], v[5])) } // Validate validates the version. @@ -151,32 +100,15 @@ func (csv *Version) Validate() error { if csv.Major < 0 { return fmt.Errorf("major version should be a non-negative integer") } - - if csv.Channel != ChannelStable { - // Check if the patch is a valid integer - if isInteger(csv.Patch) { - // If it's an integer, check if it's greater than 0 - patchInt, _ := strconv.Atoi(csv.Patch) - if patchInt < 0 { - return fmt.Errorf("patch version should be a non-negative integer") - } - } - - // If it's alpha numeric, check if it's empty - if csv.Patch == "" { - return fmt.Errorf("patch can't empty") - } + if !csv.Channel.IsValid() { + return fmt.Errorf("invalid channel: %s", csv.Channel) + } + if csv.Patch < 0 { + return fmt.Errorf("patch version should be a non-negative integer") } - return nil } -// isInteger checks if the given string is a valid integer. -func isInteger(s string) bool { - _, err := strconv.Atoi(s) - return err == nil -} - // Compare compares two Version structs // Returns 1 if csv is greater than input // Returns -1 if csv is less than input @@ -206,23 +138,9 @@ func (csv Version) Compare(input Version) (int, error) { return 0, nil } -// String converts a Version struct to a string representation. -// If the channel is stable, it returns the version in the format "vMajor". -// Otherwise, it returns the version in the format "vMajor-Channel-Patch". func (csv Version) String() string { if csv.Channel == ChannelStable { return fmt.Sprintf("v%d", csv.Major) } - return fmt.Sprintf("v%d-%s-%s", csv.Major, csv.Channel, csv.Patch) -} - -// StringWithDot converts a Version struct to a string representation. -// If the channel is stable, it returns the version in the format "vMajor". -// Otherwise, it returns the version in the format "vMajor-Channel.Patch", -// similar to String but with a dot separating channel and patch. -func (csv Version) StringWithDot() string { - if csv.Channel == ChannelStable { - return fmt.Sprintf("v%d", csv.Major) - } - return fmt.Sprintf("v%d-%s.%s", csv.Major, csv.Channel, csv.Patch) + return fmt.Sprintf("v%d-%s-%d", csv.Major, csv.Channel, csv.Patch) } diff --git a/go.mod b/go.mod index 27556e4a7..cf560ee0d 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/yaml.v3 v3.0.1 k8s.io/apiextensions-apiserver v0.27.2 k8s.io/apiserver v0.27.2 // indirect k8s.io/component-base v0.27.2 // indirect