From be8983649360d957da7f96524c07a48b41e9463e Mon Sep 17 00:00:00 2001 From: "michal.gubricky" Date: Fri, 12 Apr 2024 11:08:09 +0200 Subject: [PATCH 1/6] Support specifying the CA cert or adding verify=false into clouds.yaml Signed-off-by: michal.gubricky --- .../openstacknodeimagerelease_controller.go | 61 +++++++++++++--- ...enstacknodeimagerelease_controller_test.go | 70 +++++++++++++++++-- 2 files changed, 117 insertions(+), 14 deletions(-) diff --git a/internal/controller/openstacknodeimagerelease_controller.go b/internal/controller/openstacknodeimagerelease_controller.go index 9ac398d7..359f172c 100644 --- a/internal/controller/openstacknodeimagerelease_controller.go +++ b/internal/controller/openstacknodeimagerelease_controller.go @@ -19,7 +19,10 @@ package controller import ( "context" + "crypto/tls" + "crypto/x509" "fmt" + "net/http" "time" "github.com/gophercloud/gophercloud/v2" @@ -53,6 +56,7 @@ const ( defaultCloudName = "openstack" cloudNameSecretKey = "cloudName" cloudsSecretKey = "clouds.yaml" + caSecretKey = "cacert" waitForImageBecomeActive = 30 * time.Second ) @@ -97,7 +101,7 @@ func (r *OpenStackNodeImageReleaseReconciler) Reconcile(ctx context.Context, req }() // Get OpenStack cloud config from sercet - cloud, err := r.getCloudFromSecret(ctx, openstacknodeimagerelease.Namespace, openstacknodeimagerelease.Spec.IdentityRef.Name) + cloud, caCert, err := r.getCloudFromSecret(ctx, openstacknodeimagerelease.Namespace, openstacknodeimagerelease.Spec.IdentityRef.Name) if err != nil { if apierrors.IsNotFound(err) { conditions.MarkFalse(openstacknodeimagerelease, @@ -122,9 +126,39 @@ func (r *OpenStackNodeImageReleaseReconciler) Reconcile(ctx context.Context, req conditions.MarkTrue(openstacknodeimagerelease, apiv1alpha1.CloudAvailableCondition) + clientOpts := new(clientconfig.ClientOpts) + + if cloud.AuthInfo != nil { + clientOpts.AuthInfo = cloud.AuthInfo + clientOpts.AuthType = cloud.AuthType + clientOpts.RegionName = cloud.RegionName + clientOpts.EndpointType = cloud.EndpointType + } + opts, _ := clientconfig.AuthOptions(clientOpts) + opts.AllowReauth = true + // Create an OpenStack provider client - opts := &clientconfig.ClientOpts{AuthInfo: cloud.AuthInfo} - providerClient, err := clientconfig.AuthenticatedClient(ctx, opts) + providerClient, _ := openstack.NewClient(opts.IdentityEndpoint) + + config := &tls.Config{ + MinVersion: tls.VersionTLS12, + } + + if cloud.Verify != nil { + config.InsecureSkipVerify = !*cloud.Verify + } + + if caCert != nil { + config.RootCAs = x509.NewCertPool() + ok := config.RootCAs.AppendCertsFromPEM(caCert) + if !ok { + // If no certificates were successfully parsed, set RootCAs to nil + config.RootCAs = nil + } + } + + providerClient.HTTPClient.Transport = &http.Transport{Proxy: http.ProxyFromEnvironment, TLSClientConfig: config} + err = openstack.Authenticate(ctx, providerClient, *opts) if err != nil { record.Warnf(openstacknodeimagerelease, "OpenStackProviderClientNotSet", err.Error()) logger.Error(err, "failed to create a provider client") @@ -293,7 +327,7 @@ func (r *OpenStackNodeImageReleaseReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, nil } -func (r *OpenStackNodeImageReleaseReconciler) getCloudFromSecret(ctx context.Context, secretNamespace, secretName string) (clientconfig.Cloud, error) { +func (r *OpenStackNodeImageReleaseReconciler) getCloudFromSecret(ctx context.Context, secretNamespace, secretName string) (clientconfig.Cloud, []byte, error) { var clouds clientconfig.Clouds emptyCloud := clientconfig.Cloud{} var cloudName string @@ -304,7 +338,7 @@ func (r *OpenStackNodeImageReleaseReconciler) getCloudFromSecret(ctx context.Con Name: secretName, }, secret) if err != nil { - return emptyCloud, fmt.Errorf("failed to get secret %s in namespace %s: %w", secretName, secretNamespace, err) + return emptyCloud, nil, fmt.Errorf("failed to get secret %s in namespace %s: %w", secretName, secretNamespace, err) } content, ok := secret.Data[cloudNameSecretKey] @@ -312,23 +346,30 @@ func (r *OpenStackNodeImageReleaseReconciler) getCloudFromSecret(ctx context.Con cloudName = defaultCloudName } else { if err := yaml.Unmarshal(content, &cloudName); err != nil { - return emptyCloud, fmt.Errorf("failed to unmarshal cloudName stored in secret %s: %w", secretName, err) + return emptyCloud, nil, fmt.Errorf("failed to unmarshal cloudName stored in secret %s: %w", secretName, err) } } content, ok = secret.Data[cloudsSecretKey] if !ok { - return emptyCloud, fmt.Errorf("OpenStack credentials secret %s did not contain key %s", secretName, cloudsSecretKey) + return emptyCloud, nil, fmt.Errorf("OpenStack credentials secret %s did not contain key %s", secretName, cloudsSecretKey) } if err = yaml.Unmarshal(content, &clouds); err != nil { - return emptyCloud, fmt.Errorf("failed to unmarshal clouds credentials stored in secret %s: %w", secretName, err) + return emptyCloud, nil, fmt.Errorf("failed to unmarshal clouds credentials stored in secret %s: %w", secretName, err) } cloud, ok := clouds.Clouds[cloudName] if !ok { - return emptyCloud, fmt.Errorf("failed to find cloud %s in %s", cloudName, cloudsSecretKey) + return emptyCloud, nil, fmt.Errorf("failed to find cloud %s in %s", cloudName, cloudsSecretKey) } - return cloud, nil + + // get caCert + caCert, ok := secret.Data[caSecretKey] + if !ok { + return cloud, nil, nil + } + + return cloud, caCert, nil } func getImageID(ctx context.Context, imagesClient *gophercloud.ServiceClient, imageCreateOps *apiv1alpha1.CreateOpts) (string, error) { diff --git a/internal/controller/openstacknodeimagerelease_controller_test.go b/internal/controller/openstacknodeimagerelease_controller_test.go index a2022669..8695fca8 100644 --- a/internal/controller/openstacknodeimagerelease_controller_test.go +++ b/internal/controller/openstacknodeimagerelease_controller_test.go @@ -68,7 +68,7 @@ clouds: Client: client, } - cloud, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) + cloud, caCert, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) expectedCloud := clientconfig.Cloud{ AuthInfo: &clientconfig.AuthInfo{ @@ -83,6 +83,65 @@ clouds: } assert.NoError(t, err) assert.Equal(t, expectedCloud, cloud) + assert.Equal(t, []byte(nil), caCert) + + err = client.Delete(context.TODO(), secret) + assert.NoError(t, err) +} + +func TestGetCloudFromSecretWithCaCert(t *testing.T) { + client := fake.NewClientBuilder().Build() + + secretName := "test-secret" + secretNamespace := "test-namespace" + cloudsYAML := ` +clouds: + openstack: + auth: + username: test_user + password: test_password + project_name: test_project + project_id: test_project_id + auth_url: test_auth_url + domain_name: test_domain + region_name: test_region +` + expectedcaCert := []byte("test-ca-cert") + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: secretNamespace, + }, + Data: map[string][]byte{ + cloudsSecretKey: []byte(cloudsYAML), + caSecretKey: expectedcaCert, + }, + Type: corev1.SecretTypeOpaque, + } + err := client.Create(context.TODO(), secret) + assert.NoError(t, err) + + r := &OpenStackNodeImageReleaseReconciler{ + Client: client, + } + + cloud, caCert, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) + + expectedCloud := clientconfig.Cloud{ + AuthInfo: &clientconfig.AuthInfo{ + Username: "test_user", + Password: "test_password", + ProjectName: "test_project", + ProjectID: "test_project_id", + AuthURL: "test_auth_url", + DomainName: "test_domain", + }, + RegionName: "test_region", + } + assert.NoError(t, err) + assert.Equal(t, expectedCloud, cloud) + assert.Equal(t, caCert, expectedcaCert) err = client.Delete(context.TODO(), secret) assert.NoError(t, err) @@ -99,7 +158,7 @@ func TestGetCloudFromSecretNotFound(t *testing.T) { secretNamespace := "nonexistent-namespace" expectedError := "secrets \"nonexistent-secret\" not found" - cloud, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) + cloud, caCert, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) expectedErrorMessage := fmt.Sprintf("failed to get secret %s in namespace %s: %v", secretName, secretNamespace, expectedError) @@ -107,6 +166,7 @@ func TestGetCloudFromSecretNotFound(t *testing.T) { assert.True(t, apierrors.IsNotFound(err)) assert.Equal(t, clientconfig.Cloud{}, cloud) assert.EqualError(t, err, expectedErrorMessage) + assert.Equal(t, []byte(nil), caCert) } func TestGetCloudFromSecretMissingCloudsSecretKey(t *testing.T) { @@ -131,11 +191,12 @@ func TestGetCloudFromSecretMissingCloudsSecretKey(t *testing.T) { err := client.Create(context.TODO(), secret) assert.NoError(t, err) - cloud, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) + cloud, caCert, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) assert.Error(t, err) assert.EqualError(t, err, fmt.Sprintf("OpenStack credentials secret %s did not contain key %s", secretName, cloudsSecretKey)) assert.Equal(t, clientconfig.Cloud{}, cloud) + assert.Equal(t, []byte(nil), caCert) err = client.Delete(context.TODO(), secret) assert.NoError(t, err) @@ -178,11 +239,12 @@ clouds: err := client.Create(context.TODO(), secret) assert.NoError(t, err) - cloud, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) + cloud, caCert, err := r.getCloudFromSecret(context.TODO(), secretNamespace, secretName) assert.Error(t, err) assert.EqualError(t, err, fmt.Sprintf("failed to find cloud %s in %s", cloudName, cloudsSecretKey)) assert.Equal(t, clientconfig.Cloud{}, cloud) + assert.Equal(t, []byte(nil), caCert) err = client.Delete(context.TODO(), secret) assert.NoError(t, err) From 2de04a80515ee9b371d1d4d12c721ff41572f9fb Mon Sep 17 00:00:00 2001 From: "michal.gubricky" Date: Mon, 15 Apr 2024 16:41:26 +0200 Subject: [PATCH 2/6] Adds troubleshooting guide Signed-off-by: michal.gubricky --- docs/troubleshooting.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/troubleshooting.md diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 00000000..32b4e8b4 --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,29 @@ +# Troubleshooting + +This guide explains general info on how to debug issues if a cluster creation fails. + +## providerClient authentication err + +If you are using https, and when you encounter issue like: + +``` +kubectl logs -n cspoo-system logs -l control-plane=capo-controller-manager +... +[manager] 2024-04-15T15:20:07Z DEBUG events Post "https://10.0.3.15/identity/v3/auth/tokens": tls: failed to verify certificate: x509: certificate signed by unknown authority {"type": "Warning", "object": {"kind":"OpenStackNodeImageRelease","namespace":"cluster","name":"openstack-ferrol-1-27-ubuntu-capi-image-v1.27.8-v2","uid":"93d2c1c8-5a19-45f8-9f93-8e8bd5227ebf","apiVersion":"infrastructure.clusterstack.x-k8s.io/v1alpha1","resourceVersion":"3773"}, "reason": "OpenStackProviderClientNotSet"} +... +``` + + you must specify the CA certificate in your secret, which contains the access data to the OpenStack intance, then secret should looks similiar to this example: + + ```bash +apiVersion: v1 +data: + caCert: + clouds.yaml: +kind: Secret +metadata: + labels: + clusterctl.cluster.x-k8s.io/move: "true" + name: "openstack" + namespace: cluster + ``` From 8a716b28532f1b703d0a8f2d3b85705bd9923d97 Mon Sep 17 00:00:00 2001 From: Michal Gubricky Date: Tue, 16 Apr 2024 10:04:44 +0200 Subject: [PATCH 3/6] Update docs/troubleshooting.md Co-authored-by: Roman Hros Signed-off-by: Michal Gubricky --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 32b4e8b4..6224acea 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -4,7 +4,7 @@ This guide explains general info on how to debug issues if a cluster creation fa ## providerClient authentication err -If you are using https, and when you encounter issue like: +If you are using https, and when you encounter issues like: ``` kubectl logs -n cspoo-system logs -l control-plane=capo-controller-manager From ffe0073f4abf74d9c09929d6a27d066f8ead88b9 Mon Sep 17 00:00:00 2001 From: Michal Gubricky Date: Tue, 16 Apr 2024 10:04:49 +0200 Subject: [PATCH 4/6] Update docs/troubleshooting.md Co-authored-by: Roman Hros Signed-off-by: Michal Gubricky --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6224acea..783c3188 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -13,7 +13,7 @@ kubectl logs -n cspoo-system logs -l control-plane=capo-controller-manager Date: Tue, 16 Apr 2024 10:04:53 +0200 Subject: [PATCH 5/6] Update docs/troubleshooting.md Co-authored-by: Roman Hros Signed-off-by: Michal Gubricky --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 783c3188..c9664284 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -18,7 +18,7 @@ you must specify the CA certificate in your secret, which contains the access da ```bash apiVersion: v1 data: - caCert: + cacert: clouds.yaml: kind: Secret metadata: From 06db963264cd62b424117c4304111d775ad2188f Mon Sep 17 00:00:00 2001 From: Michal Gubricky Date: Tue, 16 Apr 2024 10:05:03 +0200 Subject: [PATCH 6/6] Update docs/troubleshooting.md Co-authored-by: Roman Hros Signed-off-by: Michal Gubricky --- docs/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c9664284..2ee0b05e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,7 +7,7 @@ This guide explains general info on how to debug issues if a cluster creation fa If you are using https, and when you encounter issues like: ``` -kubectl logs -n cspoo-system logs -l control-plane=capo-controller-manager +kubectl logs -n cspo-system -l control-plane=controller-manager ... [manager] 2024-04-15T15:20:07Z DEBUG events Post "https://10.0.3.15/identity/v3/auth/tokens": tls: failed to verify certificate: x509: certificate signed by unknown authority {"type": "Warning", "object": {"kind":"OpenStackNodeImageRelease","namespace":"cluster","name":"openstack-ferrol-1-27-ubuntu-capi-image-v1.27.8-v2","uid":"93d2c1c8-5a19-45f8-9f93-8e8bd5227ebf","apiVersion":"infrastructure.clusterstack.x-k8s.io/v1alpha1","resourceVersion":"3773"}, "reason": "OpenStackProviderClientNotSet"} ...