✨ Add timeout while evaluating CEL expression

api/v1alpha1/clusteraddon_types.go

@@ -49,6 +49,18 @@ type ClusterAddonStatus struct {
 	// +optional
 	Resources []*Resource `json:"resources,omitempty"`
 
+	// CurrentHook specifies the current running Hook.
+	// +optional
+	CurrentHook string `json:"currentHook,omitempty"`
+
+	// KubernetesVersion is the kubernetes version of the current cluster stack release.
+	// +optional
+	KubernetesVersion string `json:"kubernetesVersion,omitempty"`
+
+	// HelmChartStatus defines the status of helm chart in the cluster addon.
+	// +optional
+	HelmChartStatus map[string]HelmChartStatusConditions `json:"helmChartStatus,omitempty"`
+
 	// +optional
 	// +kubebuilder:default:=false
 	Ready bool `json:"ready"`

api/v1alpha1/conditions_const.go

@@ -26,6 +26,28 @@ const (
 	ControlPlaneNotReadyReason = "ControlPlaneNotReady"
 )
 
+const (
+	// EvaluatedCELCondition reports on whether the CEL expression is evaluated properly.
+	EvaluatedCELCondition clusterv1.ConditionType = "EvaluatedCEL"
+
+	// FailedToEvaluatePreConditionReason is used when some pre CEL expression have been failed to evaluate.
+	FailedToEvaluatePreConditionReason = "FailedToEvaluatePreCondition"
+
+	// FailedToEvaluatePostConditionReason is used when some post CEL expression have been failed to evaluate.
+	FailedToEvaluatePostConditionReason = "FailedToEvaluatePostCondition"
+
+	// CELEvaluationTimeoutReason is used when CEL expression have been failed to evaluate and timeout(1 min) has occurred.
+	CELEvaluationTimeoutReason = "CELEvaluationTimeout"
+)
+
+const (
+	// HelmChartFoundCondition reports when mentioned helm chart is present in the cluster addon tar archive.
+	HelmChartFoundCondition = "HelmChartFound"
+
+	// HelmChartMissingReason is used when mentioned helm chart is missing in the cluster addon tar archive.
+	HelmChartMissingReason = "HelmChartMissing"
+)
+
 const (
 	// HelmChartAppliedCondition reports on whether the relevant helm chart has been applied.
 	HelmChartAppliedCondition clusterv1.ConditionType = "HelmChartApplied"

config/crd/bases/clusterstack.x-k8s.io_clusteraddons.yaml

@@ -152,6 +152,18 @@ spec:
                   - type
                   type: object
                 type: array
+              currentHook:
+                description: CurrentHook specifies the current running Hook.
+                type: string
+              helmChartStatus:
+                additionalProperties: {}
+                description: HelmChartStatus defines the status of helm chart in the
+                  cluster addon.
+                type: object
+              kubernetesVersion:
+                description: KubernetesVersion is the kubernetes version of the current
+                  cluster stack release.
+                type: string
               ready:
                 default: false
                 type: boolean

extension/config/default/sa-roles.yaml

@@ -18,6 +18,7 @@ rules:
       - clusterstack.x-k8s.io
     resources:
       - clusteraddons
+      - clusteraddons/status
     verbs:
       - get
       - patch

extension/go.mod

@@ -1,6 +1,8 @@
 module github.com/SovereignCloudStack/cluster-stack-operator/extension
 
-go 1.19
+go 1.21
+
+toolchain go1.21.4
 
 replace github.com/SovereignCloudStack/cluster-stack-operator => ../
 
@@ -55,6 +57,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.25.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
 	golang.org/x/oauth2 v0.14.0 // indirect
 	golang.org/x/sys v0.14.0 // indirect

extension/go.sum

@@ -181,6 +181,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=

extension/handlers/hooks.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 
 	csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
 	"sigs.k8s.io/cluster-api/util/conditions"
@@ -47,28 +48,43 @@ func (e *ExtensionHandler) DoBeforeClusterUpgrade(ctx context.Context, request *
 	key := types.NamespacedName{Name: fmt.Sprintf("cluster-addon-%s", request.Cluster.GetName()), Namespace: request.Cluster.GetNamespace()}
 	clusterAddon := &csov1alpha1.ClusterAddon{}
 	if err := e.client.Get(ctx, key, clusterAddon); err != nil {
+		log.Error(err, "failed to get cluster addon")
 		response.SetStatus(runtimehooksv1.ResponseStatusFailure)
 		response.SetMessage(err.Error())
 		return
 	}
 
-	patchHelper, err := patch.NewHelper(clusterAddon, e.client)
-	if err != nil {
-		response.SetStatus(runtimehooksv1.ResponseStatusFailure)
-		response.SetMessage(fmt.Errorf("failed to init patch helper: %w", err).Error())
-	}
+	if clusterAddon.Spec.Hook != "BeforeClusterUpgrade" {
+		patchHelper, err := patch.NewHelper(clusterAddon, e.client)
+		if err != nil {
+			log.Error(err, "failed to create patch helper")
+			response.SetStatus(runtimehooksv1.ResponseStatusFailure)
+			response.SetMessage(err.Error())
+		}
+
+		clusterAddon.Spec.Hook = "BeforeClusterUpgrade"
+		conditions.Delete(clusterAddon, csov1alpha1.HelmChartAppliedCondition)
 
-	defer func() {
 		if err := patchHelper.Patch(ctx, clusterAddon); err != nil {
+			log.Error(err, "failed to patch cluster addon")
 			response.SetStatus(runtimehooksv1.ResponseStatusFailure)
-			response.SetMessage(fmt.Errorf("failed to patch clusterAddon: %w", err).Error())
+			response.SetMessage(err.Error())
 		}
-	}()
 
-	if conditions.IsFalse(clusterAddon, csov1alpha1.HelmChartAppliedCondition) {
-		// wait N seconds
 		response.SetRetryAfterSeconds(10)
+		return
+	}
+
+	if e.isTimeoutConditionPresent(ctx, key, clusterAddon) {
+		message := fmt.Sprintf("timeout occurred while evaluating CEL expression: %s", conditions.GetMessage(clusterAddon, csov1alpha1.EvaluatedCELCondition))
+		log.Info(message)
 		response.SetStatus(runtimehooksv1.ResponseStatusFailure)
+		response.SetMessage(message)
+		return
+	}
+
+	if !utils.IsPresentAndTrue(ctx, e.client, key, clusterAddon, csov1alpha1.HelmChartAppliedCondition) {
+		response.SetRetryAfterSeconds(5)
 		return
 	}
 
@@ -97,19 +113,36 @@ func (e *ExtensionHandler) DoAfterControlPlaneInitialized(ctx context.Context, r
 		return
 	}
 
-	patchHelper, err := patch.NewHelper(clusterAddon, e.client)
-	if err != nil {
-		response.SetStatus(runtimehooksv1.ResponseStatusFailure)
-		response.SetMessage(fmt.Errorf("failed to init patch helper: %w", err).Error())
-	}
+	if clusterAddon.Spec.Hook != "AfterControlPlaneInitialized" {
+		patchHelper, err := patch.NewHelper(clusterAddon, e.client)
+		if err != nil {
+			log.Error(err, "failed to create patch helper")
+			response.SetStatus(runtimehooksv1.ResponseStatusFailure)
+			response.SetMessage(fmt.Errorf("failed to init patch helper: %w", err).Error())
+		}
 
-	defer func() {
-		if err := patchHelper.Patch(ctx, clusterAddon); err != nil {
+		clusterAddon.Spec.Hook = "AfterControlPlaneInitialized"
+		conditions.Delete(clusterAddon, csov1alpha1.HelmChartAppliedCondition)
+
+		if err := patchHelper.Patch(ctx, clusterAddon, patch.WithForceOverwriteConditions{}); err != nil {
+			log.Error(err, "failed to patch cluster addon")
 			response.SetStatus(runtimehooksv1.ResponseStatusFailure)
 			response.SetMessage(fmt.Errorf("failed to patch clusterAddon: %w", err).Error())
 		}
-	}()
+	}
+
+	if e.isTimeoutConditionPresent(ctx, key, clusterAddon) {
+		message := fmt.Sprintf("timeout occurred while evaluating CEL expression: %s", conditions.GetMessage(clusterAddon, csov1alpha1.EvaluatedCELCondition))
+		log.Info(message)
+		response.SetStatus(runtimehooksv1.ResponseStatusFailure)
+		response.SetMessage(message)
+		return
+	}
 
 	response.SetStatus(runtimehooksv1.ResponseStatusSuccess)
 	return
 }
+
+func (e *ExtensionHandler) isTimeoutConditionPresent(ctx context.Context, key types.NamespacedName, getter conditions.Getter) bool {
+	return utils.IsPresentAndFalseWithReason(ctx, e.client, key, getter, csov1alpha1.EvaluatedCELCondition, csov1alpha1.CELEvaluationTimeoutReason)
+}