diff --git a/README.md b/README.md index 7a336fce..5209d56b 100644 --- a/README.md +++ b/README.md @@ -64,4 +64,30 @@ To tear down the kind cluster, use: $ make delete-bootstrap-cluster ``` -If you have any trouble finding the right command, then you can use `make help` to get a list of all available make targets. \ No newline at end of file +If you have any trouble finding the right command, then you can use `make help` to get a list of all available make targets. + +## Troubleshooting + +Check the latest events: + +```shell +kubectl get events -A --sort-by=.lastTimestamp +``` + +Check the conditions: + +```shell +go run github.com/guettli/check-conditions@latest all +``` + +Check with `clusterctl`: + +```shell +clusterctl describe cluster -n cluster my-cluster +``` + +Check the logs. List all logs from all deployments. Show the logs of the last ten minutes: + +```shell +kubectl get deployment -A --no-headers | while read -r ns d _; do echo; echo "====== $ns $d"; kubectl logs --since=10m -n $ns deployment/$d; done +``` diff --git a/cmd/main.go b/cmd/main.go index c9c5c8e2..4bc3da9b 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -31,6 +31,7 @@ import ( githubclient "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube" "github.com/SovereignCloudStack/cluster-stack-operator/pkg/utillog" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/workloadcluster" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -54,19 +55,23 @@ func init() { //+kubebuilder:scaffold:scheme } -func main() { - var metricsAddr string - var probeAddr string - var enableLeaderElection bool - var leaderElectionNamespace string - var watchFilterValue string - var watchNamespace string - var clusterStackConcurrency int - var clusterStackReleaseConcurrency int - var clusterAddonConcurrency int - var logLevel string - var releaseDir string +var ( + metricsAddr string + probeAddr string + enableLeaderElection bool + leaderElectionNamespace string + watchFilterValue string + watchNamespace string + clusterStackConcurrency int + clusterStackReleaseConcurrency int + clusterAddonConcurrency int + logLevel string + releaseDir string + qps float64 + burst int +) +func main() { flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":9440", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") @@ -78,6 +83,8 @@ func main() { flag.IntVar(&clusterAddonConcurrency, "clusteraddon-concurrency", 1, "Number of ClusterAddons to process simultaneously") flag.StringVar(&logLevel, "log-level", "info", "Specifies log level. Options are 'debug', 'info' and 'error'") flag.StringVar(&releaseDir, "release-dir", "/tmp/downloads/", "Specify release directory for cluster-stack releases") + flag.Float64Var(&qps, "qps", 50, "Enable custom query per second for kubernetes API server") + flag.IntVar(&burst, "burst", 100, "Enable custom burst defines how many queries the API server will accept before enforcing the limit established by qps") flag.Parse() @@ -111,6 +118,15 @@ func main() { gitFactory := githubclient.NewFactory() + restConfig := mgr.GetConfig() + restConfig.QPS = float32(qps) + restConfig.Burst = burst + + restConfigSettings := controller.RestConfigSettings{ + QPS: float32(qps), + Burst: burst, + } + var wg sync.WaitGroup wg.Add(1) @@ -123,9 +139,10 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ClusterStack") os.Exit(1) } + if err = (&controller.ClusterStackReleaseReconciler{ Client: mgr.GetClient(), - RESTConfig: mgr.GetConfig(), + RESTConfig: restConfig, ReleaseDirectory: releaseDir, WatchFilterValue: watchFilterValue, KubeClientFactory: kube.NewFactory(), @@ -135,6 +152,18 @@ func main() { os.Exit(1) } + if err = (&controller.ClusterAddonReconciler{ + Client: mgr.GetClient(), + ReleaseDirectory: releaseDir, + RestConfigSettings: restConfigSettings, + WatchFilterValue: watchFilterValue, + KubeClientFactory: kube.NewFactory(), + WorkloadClusterFactory: workloadcluster.NewFactory(), + }).SetupWithManager(ctx, mgr, controllerruntimecontroller.Options{MaxConcurrentReconciles: clusterAddonConcurrency}); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ClusterAddon") + os.Exit(1) + } + if err = (&controller.ClusterAddonCreateReconciler{ Client: mgr.GetClient(), }).SetupWithManager(ctx, mgr, controllerruntimecontroller.Options{}); err != nil { diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index c7e518ec..95e22623 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,7 +7,7 @@ commonlabels: resources: - bases/clusterstack.x-k8s.io_clusterstacks.yaml - bases/clusterstack.x-k8s.io_clusterstackreleases.yaml -#- bases/clusterstack.x-k8s.io_clusteraddons.yaml +- bases/clusterstack.x-k8s.io_clusteraddons.yaml #+kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/cso/cluster.yaml b/config/cso/cluster.yaml index fbc12e1f..dc88c05f 100644 --- a/config/cso/cluster.yaml +++ b/config/cso/cluster.yaml @@ -2,7 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: name: "${CLUSTER_NAME}" - namespace: ${NAMESPACE} + namespace: cluster spec: clusterNetwork: services: @@ -23,4 +23,4 @@ spec: machineDeployments: - class: workeramd64 name: md-0 - replicas: 1 \ No newline at end of file + replicas: 1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index eeeedf7c..47757a65 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -39,12 +39,38 @@ rules: - patch - update - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - clusters + verbs: + - get - apiGroups: - clusterstack.x-k8s.io resources: - clusteraddons verbs: - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - clusterstack.x-k8s.io + resources: + - clusteraddons/finalizers + verbs: + - update +- apiGroups: + - clusterstack.x-k8s.io + resources: + - clusteraddons/status + verbs: + - get + - patch + - update - apiGroups: - clusterstack.x-k8s.io resources: diff --git a/go.mod b/go.mod index b536b426..d0faf40f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.19 require ( github.com/go-logr/logr v1.2.4 + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 github.com/google/go-github/v52 v52.0.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.8 @@ -21,7 +22,6 @@ require ( ) require ( - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/hack/kind-dev.sh b/hack/kind-dev.sh index dc1c0699..53b5bab2 100755 --- a/hack/kind-dev.sh +++ b/hack/kind-dev.sh @@ -44,6 +44,9 @@ kindV1Alpha4Cluster: nodes: - role: control-plane image: kindest/node:${CLUSTER_VERSION} + extraMounts: + - hostPath: /var/run/docker.sock + containerPath: /var/run/docker.sock networking: podSubnet: "10.244.0.0/16" serviceSubnet: "10.96.0.0/12" diff --git a/internal/controller/clusteraddon_controller.go b/internal/controller/clusteraddon_controller.go new file mode 100644 index 00000000..d2bfe285 --- /dev/null +++ b/internal/controller/clusteraddon_controller.go @@ -0,0 +1,455 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "text/template" + "time" + + csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/kube" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/release" + "github.com/SovereignCloudStack/cluster-stack-operator/pkg/workloadcluster" + sprig "github.com/go-task/slim-sprig" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/controllers/external" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/patch" + "sigs.k8s.io/cluster-api/util/predicates" + "sigs.k8s.io/cluster-api/util/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const clusterAddonNamespace = "kube-system" + +// RestConfigSettings contains Kubernetes rest config settings. +type RestConfigSettings struct { + QPS float32 + Burst int +} + +// ClusterAddonReconciler reconciles a ClusterAddon object. +type ClusterAddonReconciler struct { + client.Client + RestConfigSettings + ReleaseDirectory string + KubeClientFactory kube.Factory + WatchFilterValue string + WorkloadClusterFactory workloadcluster.Factory +} + +//+kubebuilder:rbac:groups=clusterstack.x-k8s.io,resources=clusteraddons,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=clusterstack.x-k8s.io,resources=clusteraddons/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=clusterstack.x-k8s.io,resources=clusteraddons/finalizers,verbs=update +//+kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters,verbs=get + +// Reconcile is part of the main Kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +func (r *ClusterAddonReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, reterr error) { + clusterAddon := &csov1alpha1.ClusterAddon{} + if err := r.Get(ctx, req.NamespacedName, clusterAddon); err != nil { + if apierrors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, fmt.Errorf("failed to get cluster addon %s/%s: %w", req.Name, req.Namespace, err) + } + + patchHelper, err := patch.NewHelper(clusterAddon, r.Client) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to init patch helper: %w", err) + } + + defer func() { + conditions.SetSummary(clusterAddon) + + if err := patchHelper.Patch(ctx, clusterAddon); err != nil { + reterr = fmt.Errorf("failed to patch clusterAddon: %w", err) + } + }() + + // retrieve associated cluster object + cluster := &clusterv1.Cluster{} + clusterName := client.ObjectKey{ + Name: clusterAddon.Spec.ClusterRef.Name, + Namespace: clusterAddon.Spec.ClusterRef.Namespace, + } + + if err := r.Get(ctx, clusterName, cluster); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to find cluster %v: %w", clusterName, err) + } + + restConfigClient := r.WorkloadClusterFactory.NewClient(cluster.Name, req.Namespace, r.Client) + + restConfig, err := restConfigClient.RestConfig(ctx) + if err != nil { + conditions.MarkFalse( + clusterAddon, + csov1alpha1.ClusterReadyCondition, + csov1alpha1.ControlPlaneNotReadyReason, + clusterv1.ConditionSeverityWarning, + "kubeconfig not there (yet)", + ) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // usually this is only nil in unit tests + if restConfig != nil { + restConfig.QPS = r.RestConfigSettings.QPS + restConfig.Burst = r.RestConfigSettings.Burst + + clientSet, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to create Kubernetes interface from config: %w", err) + } + + if _, err := clientSet.Discovery().RESTClient().Get().AbsPath("/readyz").DoRaw(ctx); err != nil { + conditions.MarkFalse( + clusterAddon, + csov1alpha1.ClusterReadyCondition, + csov1alpha1.ControlPlaneNotReadyReason, + clusterv1.ConditionSeverityInfo, + "control plane not ready yet", + ) + + // wait for cluster to be ready + return reconcile.Result{RequeueAfter: 10 * time.Second}, nil + } + } + + // cluster is ready, so we set a condition and can continue as well + conditions.MarkTrue(clusterAddon, csov1alpha1.ClusterReadyCondition) + + releaseAsset, download, err := release.New(cluster.Spec.Topology.Class, r.ReleaseDirectory) + if err != nil { + conditions.MarkFalse(clusterAddon, csov1alpha1.ClusterStackReleaseAssetsReadyCondition, csov1alpha1.IssueWithReleaseAssetsReason, clusterv1.ConditionSeverityError, err.Error()) + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + if download { + conditions.MarkFalse(clusterAddon, csov1alpha1.ClusterStackReleaseAssetsReadyCondition, csov1alpha1.ReleaseAssetsNotDownloadedYetReason, clusterv1.ConditionSeverityInfo, "release assets not downloaded yet") + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + + // Check for helm charts in the release assets. If they are not present, then something went wrong. + if err := releaseAsset.CheckHelmCharts(); err != nil { + msg := fmt.Sprintf("failed to validate helm charts: %s", err.Error()) + conditions.MarkFalse( + clusterAddon, + csov1alpha1.ClusterStackReleaseAssetsReadyCondition, + csov1alpha1.IssueWithReleaseAssetsReason, + clusterv1.ConditionSeverityError, + msg, + ) + record.Warnf(clusterAddon, "ValidateHelmChartFailed", msg) + return reconcile.Result{}, nil + } + + // set downloaded condition if able to read metadata file + conditions.MarkTrue(clusterAddon, csov1alpha1.ClusterStackReleaseAssetsReadyCondition) + + // Check whether current Helm chart has been applied in the workload cluster. If not, then we need to apply the helm chart (again). + // the spec.clusterStack is only set after a Helm chart from a ClusterStack has been applied successfully. + // If it is not set, the Helm chart has never been applied. + // If it is set and does not equal the ClusterClass of the cluster, then it is outdated and has to be updated. + if clusterAddon.Spec.ClusterStack != cluster.Spec.Topology.Class { + metadata := releaseAsset.Meta + + // only apply the Helm chart again if the Helm chart version has also changed from one cluster stack release to the other + if clusterAddon.Spec.Version != metadata.Versions.Components.ClusterAddon { + clusterAddon.Status.Ready = false + + in := templateAndApplyClusterAddonInput{ + clusterAddonChartPath: releaseAsset.ClusterAddonChartPath(), + clusterAddonValuesPath: releaseAsset.ClusterAddonValuesPath(), + clusterAddon: clusterAddon, + cluster: cluster, + restConfig: restConfig, + } + + shouldRequeue, err := r.templateAndApplyClusterAddonHelmChart(ctx, in) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to apply helm chart: %w", err) + } + if shouldRequeue { + // set latest version and requeue + clusterAddon.Spec.Version = metadata.Versions.Components.ClusterAddon + clusterAddon.Spec.ClusterStack = cluster.Spec.Topology.Class + + // set condition to false as we have not successfully applied Helm chart + conditions.MarkFalse( + clusterAddon, + csov1alpha1.HelmChartAppliedCondition, + csov1alpha1.FailedToApplyObjectsReason, + clusterv1.ConditionSeverityInfo, + "failed to successfully apply everything", + ) + return ctrl.Result{RequeueAfter: 20 * time.Second}, nil + } + + // Helm chart has been applied successfully + clusterAddon.Spec.Version = metadata.Versions.Components.ClusterAddon + conditions.MarkTrue(clusterAddon, csov1alpha1.HelmChartAppliedCondition) + } + + clusterAddon.Spec.ClusterStack = cluster.Spec.Topology.Class + clusterAddon.Status.Ready = true + return ctrl.Result{}, nil + } + + // if condition is false we have not yet successfully applied the helm chart + if conditions.IsFalse(clusterAddon, csov1alpha1.HelmChartAppliedCondition) { + in := templateAndApplyClusterAddonInput{ + clusterAddonChartPath: releaseAsset.ClusterAddonChartPath(), + clusterAddonValuesPath: releaseAsset.ClusterAddonValuesPath(), + clusterAddon: clusterAddon, + cluster: cluster, + restConfig: restConfig, + } + + shouldRequeue, err := r.templateAndApplyClusterAddonHelmChart(ctx, in) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to apply helm chart: %w", err) + } + if shouldRequeue { + // set condition to false as we have not yet successfully applied helm chart + conditions.MarkFalse(clusterAddon, csov1alpha1.HelmChartAppliedCondition, csov1alpha1.FailedToApplyObjectsReason, clusterv1.ConditionSeverityInfo, "failed to successfully apply everything") + return ctrl.Result{RequeueAfter: 20 * time.Second}, nil + } + + // set condition that helm chart has been applied successfully + conditions.MarkTrue(clusterAddon, csov1alpha1.HelmChartAppliedCondition) + } + + clusterAddon.Status.Ready = true + return ctrl.Result{}, nil +} + +type templateAndApplyClusterAddonInput struct { + clusterAddonChartPath string + clusterAddonValuesPath string + clusterAddon *csov1alpha1.ClusterAddon + cluster *clusterv1.Cluster + restConfig *rest.Config +} + +func (r *ClusterAddonReconciler) templateAndApplyClusterAddonHelmChart(ctx context.Context, in templateAndApplyClusterAddonInput) (bool, error) { + clusterAddonChart := in.clusterAddonChartPath + var shouldRequeue bool + + buildTemplate, err := buildTemplateFromClusterAddonValues(ctx, in.clusterAddonValuesPath, in.cluster, r.Client) + if err != nil { + return false, fmt.Errorf("failed to build template from cluster addon values: %w", err) + } + + helmTemplate, err := helmTemplateClusterAddon(clusterAddonChart, buildTemplate) + if err != nil { + return false, fmt.Errorf("failed to template helm chart: %w", err) + } + + kubeClient := r.KubeClientFactory.NewClient(clusterAddonNamespace, in.restConfig) + + newResources, shouldRequeue, err := kubeClient.Apply(ctx, helmTemplate, in.clusterAddon.Status.Resources) + if err != nil { + return false, fmt.Errorf("failed to apply objects from cluster addon Helm chart: %w", err) + } + + in.clusterAddon.Status.Resources = newResources + return shouldRequeue, nil +} + +func buildTemplateFromClusterAddonValues(ctx context.Context, addonValuePath string, cluster *clusterv1.Cluster, c client.Client) ([]byte, error) { + data, err := os.ReadFile(filepath.Clean(addonValuePath)) + if err != nil { + return nil, fmt.Errorf("failed to read the file %s: %w", addonValuePath, err) + } + + references := map[string]corev1.ObjectReference{ + "Cluster": { + APIVersion: cluster.APIVersion, + Kind: cluster.Kind, + Namespace: cluster.Namespace, + Name: cluster.Name, + }, + } + + if cluster.Spec.ControlPlaneRef != nil { + references["ControlPlane"] = *cluster.Spec.ControlPlaneRef + } + + if cluster.Spec.InfrastructureRef != nil { + references["InfraCluster"] = *cluster.Spec.InfrastructureRef + } + + valueLookUp, err := initializeBuiltins(ctx, c, references, cluster) + if err != nil { + return nil, fmt.Errorf("failed to initialize builtins: %w", err) + } + + tmpl, err := template.New("cluster-addon-values" + "-" + cluster.GetName()). + Funcs(sprig.TxtFuncMap()). + Parse(string(data)) + if err != nil { + return nil, fmt.Errorf("failed to create new template: %w", err) + } + + var buffer bytes.Buffer + + if err := tmpl.Execute(&buffer, valueLookUp); err != nil { + return nil, fmt.Errorf("failed to execute template string %q on cluster %q: %w", string(data), cluster.GetName(), err) + } + + expandedTemplate := buffer.String() + var unmarhallData interface{} + err = yaml.Unmarshal([]byte(expandedTemplate), &unmarhallData) + if err != nil { + return nil, fmt.Errorf("unmarshal expanded template: %w", err) + } + + values, ok := unmarhallData.(map[interface{}]interface{})["values"].(string) + if !ok { + return nil, fmt.Errorf("key 'values' not found in template of cluster addon helm chart") + } + + return []byte(values), nil +} + +// helmTemplateClusterAddon takes the helm chart path and cluster addon values and generates template yaml file +// in the same path as the chart. +// Then it returns the path of the generated yaml file. +// Example: helm template /tmp/downloads/cluster-stacks/myprovider-myclusterstack-1-26-v2/myprovider-myclusterstack-1-26-v2.tgz +// The return yaml file path will be /tmp/downloads/cluster-stacks/myprovider-myclusterstack-1-26-v2/myprovider-myclusterstack-1-26-v2.tgz.yaml. +func helmTemplateClusterAddon(chartPath string, helmTemplate []byte) ([]byte, error) { + helmCommand := "helm" + helmArgs := []string{"template"} + + input := bytes.NewBuffer(helmTemplate) + + var cmdOutput bytes.Buffer + + helmArgs = append(helmArgs, "cluster-addon", filepath.Base(chartPath), "--namespace", clusterAddonNamespace, "-f", "-") + helmTemplateCmd := exec.Command(helmCommand, helmArgs...) + helmTemplateCmd.Stderr = os.Stderr + helmTemplateCmd.Dir = filepath.Dir(chartPath) + helmTemplateCmd.Stdout = &cmdOutput + helmTemplateCmd.Stdin = input + + if err := helmTemplateCmd.Run(); err != nil { + return nil, fmt.Errorf("failed to run helm template for %q: %w", chartPath, err) + } + + return cmdOutput.Bytes(), nil +} + +// initializeBuiltins takes a map of keys to object references, attempts to get the referenced objects, and returns a map of keys to the actual objects. +// These objects are a map[string]interface{} so that they can be used as values in the template. +func initializeBuiltins(ctx context.Context, c client.Client, referenceMap map[string]corev1.ObjectReference, cluster *clusterv1.Cluster) (map[string]interface{}, error) { + valueLookUp := make(map[string]interface{}) + + for name, ref := range referenceMap { + objectRef := referenceMap[name] + obj, err := external.Get(ctx, c, &objectRef, cluster.Namespace) + if err != nil { + return nil, fmt.Errorf("failed to get object %s: %w", ref.Name, err) + } + valueLookUp[name] = obj.Object + } + + return valueLookUp, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ClusterAddonReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, options controller.Options) error { + logger := ctrl.LoggerFrom(ctx) + c, err := ctrl.NewControllerManagedBy(mgr). + WithOptions(options). + For(&csov1alpha1.ClusterAddon{}). + WithEventFilter(predicates.ResourceNotPausedAndHasFilterLabel(logger, r.WatchFilterValue)). + Build(r) + if err != nil { + return fmt.Errorf("error creating controller: %w", err) + } + + // check also for updates in cluster objects + if err := c.Watch( + source.Kind(mgr.GetCache(), &clusterv1.Cluster{}), + handler.EnqueueRequestsFromMapFunc(clusterToClusterAddon(ctx)), + predicate.Funcs{ + // We're only interested in the update events for a cluster object where cluster.spec.topology.class changed + UpdateFunc: func(e event.UpdateEvent) bool { + oldCluster, ok := e.ObjectOld.(*clusterv1.Cluster) + if !ok { + return false + } + + newCluster, ok := e.ObjectNew.(*clusterv1.Cluster) + if !ok { + return false + } + + if oldCluster.Spec.Topology != nil && newCluster.Spec.Topology != nil && + oldCluster.Spec.Topology.Class != newCluster.Spec.Topology.Class { + return true + } + + return false + }, + GenericFunc: func(e event.GenericEvent) bool { + return false + }, + CreateFunc: func(e event.CreateEvent) bool { + return false + }, + DeleteFunc: func(e event.DeleteEvent) bool { + return false + }, + }, + ); err != nil { + return fmt.Errorf("failed adding a watch for ready clusters: %w", err) + } + + return nil +} + +// clusterToClusterAddon enqueues requests for clusterAddons on change of cluster objects. +func clusterToClusterAddon(_ context.Context) handler.MapFunc { + return func(ctx context.Context, o client.Object) []reconcile.Request { + clusterAddonName := types.NamespacedName{ + Namespace: o.GetNamespace(), + Name: fmt.Sprintf("cluster-addon-%s", o.GetName()), + } + + return []reconcile.Request{{NamespacedName: clusterAddonName}} + } +} diff --git a/internal/controller/clusterstackrelease_controller.go b/internal/controller/clusterstackrelease_controller.go index 4c26f621..030a003d 100644 --- a/internal/controller/clusterstackrelease_controller.go +++ b/internal/controller/clusterstackrelease_controller.go @@ -296,10 +296,7 @@ func (r *ClusterStackReleaseReconciler) templateAndApply(ctx context.Context, re // templateClusterClassHelmChart templates the clusterClass helm chart. func (*ClusterStackReleaseReconciler) templateClusterClassHelmChart(releaseAssets *release.Release, name, namespace string) ([]byte, error) { - clusterClassChart, err := releaseAssets.ClusterClassChartPath() - if err != nil { - return nil, fmt.Errorf("failed to get clusterClass helm chart path: %w", err) - } + clusterClassChart := releaseAssets.ClusterClassChartPath() splittedName := strings.Split(name, clusterstack.Separator) releaseName := strings.Join(splittedName[0:4], clusterstack.Separator) diff --git a/pkg/release/release.go b/pkg/release/release.go index bb76e3aa..6777e76d 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -77,27 +77,70 @@ func New(tag, downloadPath string) (Release, bool, error) { return Release{}, false, fmt.Errorf("couldn't verify the download path %s with error: %w", downloadPath, err) } + metadata, err := ensureMetadata(downloadPath, metadataFileName) + if err != nil { + return Release{}, false, fmt.Errorf("failed to get metadata: %w", err) + } + + rel.Meta = metadata + + // release object is populated, we can validate it now. + if err := rel.Validate(); err != nil { + return Release{}, false, fmt.Errorf("failed to validate release: %w", err) + } + + return rel, false, nil +} + +func ensureMetadata(downloadPath, metadataFileName string) (Metadata, error) { // Read the metadata.yaml file from the release. metadataPath := filepath.Join(downloadPath, metadataFileName) f, err := os.ReadFile(filepath.Clean(filepath.Join(downloadPath, metadataFileName))) if err != nil { - return Release{}, false, fmt.Errorf("failed to read metadata file %s: %w", metadataPath, err) + return Metadata{}, fmt.Errorf("failed to read metadata file %s: %w", metadataPath, err) } metadata := Metadata{} // if unmarshal fails, it indicates incomplete metadata file. // But we don't want to enforce download again. if err = yaml.Unmarshal(f, &metadata); err != nil { - return Release{}, false, fmt.Errorf("failed to unmarshal metadata: %w", err) + return Metadata{}, fmt.Errorf("failed to unmarshal metadata: %w", err) } - rel.Meta = metadata + return metadata, nil +} - if err := rel.Validate(); err != nil { - return Release{}, false, fmt.Errorf("failed to validate release: %w", err) +// CheckHelmCharts checks all expected helm charts in the release directory. +// This is a separate method, since few controllers need to check for the +// presence of helm charts and few don't. +func (r *Release) CheckHelmCharts() error { + // check if the cluster class chart is present. + clusterClassChartName := r.clusterClassChartName() + clusterClassChartPath, err := r.helmChartNamePath(clusterClassChartName) + if err != nil { + return fmt.Errorf("failed to get cluster class chart path: %w", err) + } + if _, err := os.Stat(clusterClassChartPath); err != nil { + return fmt.Errorf("failed to verify the cluster addon chart path %s with error: %w", clusterClassChartPath, err) } - return rel, false, nil + // check if the cluster addon values file is present. + valuesPath := filepath.Join(r.LocalDownloadPath, clusterAddonValuesName) + if _, err := os.Stat(valuesPath); err != nil { + return fmt.Errorf("failed to verify the cluster addon values path %s with error: %w", valuesPath, err) + } + + // check if the cluster class chart is present. + clusterAddonChartName := r.clusterAddonChartName() + clusterAddonChartPath, err := r.helmChartNamePath(clusterAddonChartName) + if err != nil { + return fmt.Errorf("failed to get cluster addon chart path: %w", err) + } + if _, err := os.Stat(clusterAddonChartPath); err != nil { + return fmt.Errorf("failed to verify the cluster class chart path %s with error: %w", clusterAddonChartPath, err) + } + + return nil } // Validate validates the release. @@ -126,8 +169,10 @@ func (r *Release) clusterAddonChartName() string { } // ClusterAddonChartPath returns the helm chart name from the given path. -func (r *Release) ClusterAddonChartPath() (string, error) { - return r.helmChartNamePath(r.clusterAddonChartName()) +func (r *Release) ClusterAddonChartPath() string { + // we ignore the error here, since we already checked for the presence of the chart. + path, _ := r.helmChartNamePath(r.clusterAddonChartName()) + return path } // ClusterAddonValuesPath returns the path to the cluster addon values file. @@ -141,9 +186,11 @@ func (r *Release) clusterClassChartName() string { } // ClusterClassChartPath returns the absolute helm chart path for cluster class. -func (r *Release) ClusterClassChartPath() (string, error) { +func (r *Release) ClusterClassChartPath() string { nameFilter := r.clusterClassChartName() - return r.helmChartNamePath(nameFilter) + // we ignore the error here, since we already checked for the presence of the chart. + path, _ := r.helmChartNamePath(nameFilter) + return path } // helmChartNamePath returns the helm chart name from the given path. diff --git a/pkg/workloadcluster/restconfig.go b/pkg/workloadcluster/restconfig.go new file mode 100644 index 00000000..e3b66d56 --- /dev/null +++ b/pkg/workloadcluster/restconfig.go @@ -0,0 +1,87 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package workloadcluster includes implementation of workload cluster interfaces. +package workloadcluster + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/cluster-api/util/secret" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type restClient struct { + controllerClient client.Client + clusterName string + namespace string +} + +// Client includes method to get rest config for a workload cluster. +type Client interface { + RestConfig(ctx context.Context) (*rest.Config, error) +} + +// Factory interface includes methods for gettong a rest client. +type Factory interface { + NewClient(name, namespace string, controllerClient client.Client) Client +} + +type factory struct{} + +// NewClient creates new workload cluster clients. +func (*factory) NewClient(name, namespace string, controllerClient client.Client) Client { + return &restClient{clusterName: name, namespace: namespace, controllerClient: controllerClient} +} + +var _ = Factory(&factory{}) + +// NewFactory creates a new factory for workload cluster clients. +func NewFactory() Factory { + return &factory{} +} + +var _ Client = &restClient{} + +func (r *restClient) RestConfig(ctx context.Context) (*rest.Config, error) { + namespacedName := types.NamespacedName{ + Name: fmt.Sprintf("%s-%s", r.clusterName, secret.Kubeconfig), + Namespace: r.namespace, + } + restSecret := &corev1.Secret{} + if err := r.controllerClient.Get(ctx, namespacedName, restSecret); err != nil { + return nil, fmt.Errorf("failed to get secret from management cluster in %s/%s: %w", namespacedName.Name, namespacedName.Namespace, err) + } + kubeconfig := string(restSecret.Data["value"]) + + clientCfg, err := clientcmd.NewClientConfigFromBytes([]byte(kubeconfig)) + if err != nil { + return nil, fmt.Errorf("failed to create client config from secret: %w", err) + } + + // Get the rest config from the kubeconfig + restCfg, err := clientCfg.ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to get rest config from client config: %w", err) + } + + return restCfg, nil +}