diff --git a/drivers/storage/portworx/component/autopilot.go b/drivers/storage/portworx/component/autopilot.go index 6e724a8ef..b8af2cabf 100644 --- a/drivers/storage/portworx/component/autopilot.go +++ b/drivers/storage/portworx/component/autopilot.go @@ -307,7 +307,7 @@ func (c *autopilot) createDeployment( sort.Strings(argList) command := append([]string{"/autopilot"}, argList...) - imageName = util.GetImageURN(cluster.Spec.CustomImageRegistry, imageName) + imageName = util.GetImageURN(cluster, imageName) envMap := make(map[string]*v1.EnvVar) envMap[pxutil.EnvKeyPortworxNamespace] = &v1.EnvVar{ diff --git a/drivers/storage/portworx/component/csi.go b/drivers/storage/portworx/component/csi.go index a66fead12..e237b49e3 100644 --- a/drivers/storage/portworx/component/csi.go +++ b/drivers/storage/portworx/component/csi.go @@ -403,24 +403,24 @@ func (c *csi) createDeployment( ) provisionerImage = util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.CSIProvisioner, ) if csiConfig.IncludeAttacher && cluster.Status.DesiredImages.CSIAttacher != "" { attacherImage = util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.CSIAttacher, ) } if csiConfig.IncludeSnapshotter && cluster.Status.DesiredImages.CSISnapshotter != "" { snapshotterImage = util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.CSISnapshotter, ) } if csiConfig.IncludeResizer && cluster.Status.DesiredImages.CSIResizer != "" { resizerImage = util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.CSIResizer, ) } @@ -695,11 +695,11 @@ func (c *csi) createStatefulSet( ) provisionerImage = util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.CSIProvisioner, ) attacherImage = util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.CSIAttacher, ) diff --git a/drivers/storage/portworx/component/lighthouse.go b/drivers/storage/portworx/component/lighthouse.go index d3fa86e31..aed32f5d3 100644 --- a/drivers/storage/portworx/component/lighthouse.go +++ b/drivers/storage/portworx/component/lighthouse.go @@ -328,10 +328,9 @@ func (c *lighthouse) createDeployment( } } - imageRegistry := cluster.Spec.CustomImageRegistry - lhImage = util.GetImageURN(imageRegistry, lhImage) - configSyncImage = util.GetImageURN(imageRegistry, configSyncImage) - storkConnectorImage = util.GetImageURN(imageRegistry, storkConnectorImage) + lhImage = util.GetImageURN(cluster, lhImage) + configSyncImage = util.GetImageURN(cluster, configSyncImage) + storkConnectorImage = util.GetImageURN(cluster, storkConnectorImage) modified := lhImage != existingLhImage || configSyncImage != existingConfigInitImage || diff --git a/drivers/storage/portworx/component/portworx_api.go b/drivers/storage/portworx/component/portworx_api.go index da50d9a2e..21c5ec858 100644 --- a/drivers/storage/portworx/component/portworx_api.go +++ b/drivers/storage/portworx/component/portworx_api.go @@ -158,7 +158,7 @@ func (c *portworxAPI) createDaemonSet( existingImageName = existingDaemonSet.Spec.Template.Spec.Containers[0].Image } - imageName := util.GetImageURN(cluster.Spec.CustomImageRegistry, pxutil.ImageNamePause) + imageName := util.GetImageURN(cluster, pxutil.ImageNamePause) serviceAccount := pxutil.PortworxServiceAccountName(cluster) existingServiceAccount := existingDaemonSet.Spec.Template.Spec.ServiceAccountName diff --git a/drivers/storage/portworx/component/portworx_proxy.go b/drivers/storage/portworx/component/portworx_proxy.go index 57b2ec33a..7b46ae379 100644 --- a/drivers/storage/portworx/component/portworx_proxy.go +++ b/drivers/storage/portworx/component/portworx_proxy.go @@ -214,7 +214,7 @@ func (c *portworxProxy) createDaemonSet( existingImageName = existingDaemonSet.Spec.Template.Spec.Containers[0].Image } - imageName := util.GetImageURN(cluster.Spec.CustomImageRegistry, "k8s.gcr.io/pause:3.1") + imageName := util.GetImageURN(cluster, "k8s.gcr.io/pause:3.1") modified := existingImageName != imageName || util.HasPullSecretChanged(cluster, existingDaemonSet.Spec.Template.Spec.ImagePullSecrets) || diff --git a/drivers/storage/portworx/component/prometheus.go b/drivers/storage/portworx/component/prometheus.go index 082a84bcb..8f6f02415 100644 --- a/drivers/storage/portworx/component/prometheus.go +++ b/drivers/storage/portworx/component/prometheus.go @@ -390,7 +390,7 @@ func (c *prometheus) createOperatorDeployment( } imageName := util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.PrometheusOperator, ) @@ -421,11 +421,11 @@ func getPrometheusOperatorDeploymentSpec( "k8s-app": PrometheusOperatorDeploymentName, } configReloaderImageName := util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.PrometheusConfigMapReload, ) prometheusConfigReloaderImageName := util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.PrometheusConfigReloader, ) args := make([]string, 0) @@ -540,7 +540,7 @@ func (c *prometheus) createPrometheusInstance( ) error { replicas := int32(1) prometheusImageName := util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, cluster.Status.DesiredImages.Prometheus, ) diff --git a/drivers/storage/portworx/component/pvccontroller.go b/drivers/storage/portworx/component/pvccontroller.go index a35af1d58..b7b7fb27b 100644 --- a/drivers/storage/portworx/component/pvccontroller.go +++ b/drivers/storage/portworx/component/pvccontroller.go @@ -275,7 +275,7 @@ func (c *pvcController) createDeployment( kubeControllerImage = "k8s.gcr.io/kube-controller-manager-amd64" } imageName := util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, kubeControllerImage+":v"+c.k8sVersion.String(), ) diff --git a/drivers/storage/portworx/deployment.go b/drivers/storage/portworx/deployment.go index 95c083432..52f7821ae 100644 --- a/drivers/storage/portworx/deployment.go +++ b/drivers/storage/portworx/deployment.go @@ -427,7 +427,7 @@ func configureStorageNodeSpec(node *corev1.StorageNode, config *cloudstorage.Con } func (t *template) portworxContainer() v1.Container { - pxImage := util.GetImageURN(t.cluster.Spec.CustomImageRegistry, t.cluster.Spec.Image) + pxImage := util.GetImageURN(t.cluster, t.cluster.Spec.Image) return v1.Container{ Name: pxContainerName, Image: pxImage, @@ -464,7 +464,7 @@ func (t *template) portworxContainer() v1.Container { } func (t *template) kvdbContainer() v1.Container { - kvdbProxyImage := util.GetImageURN(t.cluster.Spec.CustomImageRegistry, pxutil.ImageNamePause) + kvdbProxyImage := util.GetImageURN(t.cluster, pxutil.ImageNamePause) kvdbTargetPort := 9019 if t.startPort != pxutil.DefaultStartPort { kvdbTargetPort = t.startPort + 15 @@ -530,7 +530,7 @@ func (t *template) csiRegistrarContainer() *v1.Container { if t.cluster.Status.DesiredImages.CSINodeDriverRegistrar != "" { container.Name = "csi-node-driver-registrar" container.Image = util.GetImageURN( - t.cluster.Spec.CustomImageRegistry, + t.cluster, t.cluster.Status.DesiredImages.CSINodeDriverRegistrar, ) container.Args = []string{ @@ -541,7 +541,7 @@ func (t *template) csiRegistrarContainer() *v1.Container { } else if t.cluster.Status.DesiredImages.CSIDriverRegistrar != "" { container.Name = "csi-driver-registrar" container.Image = util.GetImageURN( - t.cluster.Spec.CustomImageRegistry, + t.cluster, t.cluster.Status.DesiredImages.CSIDriverRegistrar, ) container.Args = []string{ @@ -562,7 +562,7 @@ func (t *template) telemetryContainer() *v1.Container { container := v1.Container{ Name: "telemetry", Image: util.GetImageURN( - t.cluster.Spec.CustomImageRegistry, + t.cluster, t.getDesiredTelemetryImage(t.cluster), ), ImagePullPolicy: t.imagePullPolicy, diff --git a/drivers/storage/portworx/uninstall.go b/drivers/storage/portworx/uninstall.go index 8bd770549..c1b7176cd 100644 --- a/drivers/storage/portworx/uninstall.go +++ b/drivers/storage/portworx/uninstall.go @@ -180,7 +180,7 @@ func (u *uninstallPortworx) RunNodeWiper( release := manifest.Instance().GetVersions(u.cluster, true) wiperImage = release.Components.NodeWiper } - wiperImage = util.GetImageURN(u.cluster.Spec.CustomImageRegistry, wiperImage) + wiperImage = util.GetImageURN(u.cluster, wiperImage) args := []string{"-w"} if removeData { diff --git a/pkg/constants/metadata.go b/pkg/constants/metadata.go index 48808b18d..2bd2b78ea 100644 --- a/pkg/constants/metadata.go +++ b/pkg/constants/metadata.go @@ -28,6 +28,12 @@ const ( // AnnotationPodSafeToEvict annotation tells cluster autoscaler whether the // pod is safe to be evicted when scaling down a node AnnotationPodSafeToEvict = "cluster-autoscaler.kubernetes.io/safe-to-evict" + + // AnnotationCommonImageRegistries annotation contains the common image registries, separated by comma. + // When custom image registry is provided, we will replace any image with common registry with + // the custom registry, there is a list of hardcoded common registries, however the list + // may not be complete, users can use this annotation to add more. + AnnotationCommonImageRegistries = OperatorPrefix + "/common-image-registries" ) const ( diff --git a/pkg/controller/storagecluster/stork.go b/pkg/controller/storagecluster/stork.go index 528f99363..150ee4bbc 100644 --- a/pkg/controller/storagecluster/stork.go +++ b/pkg/controller/storagecluster/stork.go @@ -548,7 +548,7 @@ func (c *Controller) createStorkDeployment( sort.Strings(argList) command := append([]string{"/stork"}, argList...) - imageName = util.GetImageURN(cluster.Spec.CustomImageRegistry, imageName) + imageName = util.GetImageURN(cluster, imageName) hostNetwork := cluster.Spec.Stork.HostNetwork != nil && *cluster.Spec.Stork.HostNetwork envMap := c.Driver.GetStorkEnvMap(cluster) @@ -764,7 +764,7 @@ func (c *Controller) createStorkSchedDeployment( kubeSchedImage = "k8s.gcr.io/kube-scheduler-amd64" } imageName := util.GetImageURN( - cluster.Spec.CustomImageRegistry, + cluster, kubeSchedImage+":v"+c.kubernetesVersion.String(), ) diff --git a/pkg/util/test/util.go b/pkg/util/test/util.go index 4cf7c4b00..60a7c982f 100644 --- a/pkg/util/test/util.go +++ b/pkg/util/test/util.go @@ -686,7 +686,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl storkImageName = cluster.Spec.Stork.Image } - storkImage := util.GetImageURN(cluster.Spec.CustomImageRegistry, storkImageName) + storkImage := util.GetImageURN(cluster, storkImageName) err := validateImageOnPods(storkImage, cluster.Namespace, map[string]string{"name": "stork"}) if err != nil { return err @@ -723,7 +723,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl autopilotImageName = cluster.Spec.Autopilot.Image } - autopilotImage := util.GetImageURN(cluster.Spec.CustomImageRegistry, autopilotImageName) + autopilotImage := util.GetImageURN(cluster, autopilotImageName) if err = validateImageOnPods(autopilotImage, cluster.Namespace, map[string]string{"name": "autopilot"}); err != nil { return err } @@ -748,7 +748,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl lighthouseImageName = cluster.Spec.UserInterface.Image } - lhImage := util.GetImageURN(cluster.Spec.CustomImageRegistry, lighthouseImageName) + lhImage := util.GetImageURN(cluster, lighthouseImageName) if err = validateImageOnPods(lhImage, cluster.Namespace, map[string]string{"name": "lighthouse"}); err != nil { return err } diff --git a/pkg/util/util.go b/pkg/util/util.go index 1a418d1a2..a49d843f2 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -2,6 +2,7 @@ package util import ( "fmt" + "github.com/libopenstorage/operator/pkg/constants" "path" "reflect" "strings" @@ -39,12 +40,35 @@ var ( } ) +func getMergedCommonRegistries(cluster *corev1.StorageCluster) map[string]bool { + val, ok := cluster.Annotations[constants.AnnotationCommonImageRegistries] + + if !ok { + return commonDockerRegistries + } + + mergedCommonRegistries := make(map[string]bool) + + for _, v := range strings.Split(strings.TrimSpace(val), ",") { + mergedCommonRegistries[v] = true + } + + for k, v := range commonDockerRegistries { + mergedCommonRegistries[k] = v + } + + return mergedCommonRegistries +} + // GetImageURN returns the complete image name based on the registry and repo -func GetImageURN(registryAndRepo, image string) string { +func GetImageURN(cluster *corev1.StorageCluster, image string) string { if image == "" { return "" } + registryAndRepo := cluster.Spec.CustomImageRegistry + mergedCommonRegistries := getMergedCommonRegistries(cluster) + omitRepo := false if strings.HasSuffix(registryAndRepo, "//") { omitRepo = true @@ -59,7 +83,7 @@ func GetImageURN(registryAndRepo, image string) string { imgParts := strings.Split(image, "/") if len(imgParts) > 1 { // advance imgParts to swallow the common registry - if _, present := commonDockerRegistries[imgParts[0]]; present { + if _, present := mergedCommonRegistries[imgParts[0]]; present { imgParts = imgParts[1:] } } diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 2859dc874..0da3062eb 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1,6 +1,7 @@ package util import ( + "github.com/libopenstorage/operator/pkg/constants" "testing" "github.com/stretchr/testify/require" @@ -9,74 +10,104 @@ import ( "github.com/libopenstorage/operator/pkg/util/k8s" ) -func TestGetImageURN(t *testing.T) { +func TestImageURN(t *testing.T) { // TestCase: Empty image - out := GetImageURN("registry.io", "") + out := getImageURN("", "registry.io", "") require.Equal(t, "", out) // TestCase: Empty repo and registry - out = GetImageURN("", "test/image") + out = getImageURN("", "", "test/image") require.Equal(t, "test/image", out) // TestCase: Registry without repo but image with repo - out = GetImageURN("registry.io", "test/image") + out = getImageURN("", "registry.io", "test/image") require.Equal(t, "registry.io/test/image", out) - out = GetImageURN("registry.io/", "test/image") + out = getImageURN("", "registry.io/", "test/image") require.Equal(t, "registry.io/test/image", out) - out = GetImageURN("registry.io", "test/this/image") + out = getImageURN("", "registry.io", "test/this/image") require.Equal(t, "registry.io/test/this/image", out) // TestCase: Registry and image without repo - out = GetImageURN("registry.io", "image") + out = getImageURN("", "registry.io", "image") require.Equal(t, "registry.io/image", out) // TestCase: Image with common docker registries - out = GetImageURN("registry.io", "docker.io/test/image") + out = getImageURN("", "registry.io", "docker.io/test/image") require.Equal(t, "registry.io/test/image", out) - out = GetImageURN("registry.io", "quay.io/test/this/image") + out = getImageURN("", "registry.io", "quay.io/test/this/image") require.Equal(t, "registry.io/test/this/image", out) - out = GetImageURN("registry.io/", "index.docker.io/test/this/image") + out = getImageURN("", "registry.io/", "index.docker.io/test/this/image") require.Equal(t, "registry.io/test/this/image", out) - out = GetImageURN("registry.io", "registry-1.docker.io/image") + out = getImageURN("", "registry.io", "registry-1.docker.io/image") require.Equal(t, "registry.io/image", out) - out = GetImageURN("registry.io/", "registry.connect.redhat.com/image") + out = getImageURN("", "registry.io/", "registry.connect.redhat.com/image") require.Equal(t, "registry.io/image", out) // TestCase: Regsitry and image both with repo - out = GetImageURN("registry.io/repo", "test/image") + out = getImageURN("", "registry.io/repo", "test/image") require.Equal(t, "registry.io/repo/image", out) - out = GetImageURN("registry.io/repo", "test/this/image") + out = getImageURN("", "registry.io/repo", "test/this/image") require.Equal(t, "registry.io/repo/image", out) - out = GetImageURN("registry.io/repo/", "test/image") + out = getImageURN("", "registry.io/repo/", "test/image") require.Equal(t, "registry.io/repo/image", out) - out = GetImageURN("registry.io/repo//", "test/this/image") + out = getImageURN("", "registry.io/repo//", "test/this/image") require.Equal(t, "registry.io/repo/image", out) // TestCase: Regsitry with repo but image without repo - out = GetImageURN("registry.io/repo", "image") + out = getImageURN("", "registry.io/repo", "image") require.Equal(t, "registry.io/repo/image", out) - out = GetImageURN("registry.io/repo/subdir", "image") + out = getImageURN("", "registry.io/repo/subdir", "image") require.Equal(t, "registry.io/repo/subdir/image", out) // TestCase: Registry with empty root repo - out = GetImageURN("registry.io//", "image") + out = getImageURN("", "registry.io//", "image") require.Equal(t, "registry.io/image", out) - out = GetImageURN("registry.io//", "test/image") + out = getImageURN("", "registry.io//", "test/image") require.Equal(t, "registry.io/image", out) - out = GetImageURN("registry.io//", "test/this/image") + out = getImageURN("", "registry.io//", "test/this/image") require.Equal(t, "registry.io/image", out) + + out = getImageURN("k8s.gcr.io", "registry.io//", "k8s.gcr.io/pause:3.1") + require.Equal(t, "registry.io/pause:3.1", out) + + // Update it again, now k8s.gcr.io should be deleted from common registries. + out = getImageURN("gcr.io", "registry.io", "k8s.gcr.io/pause:3.1") + require.Equal(t, "registry.io/k8s.gcr.io/pause:3.1", out) + + out = getImageURN("", "registry.io//", "k8s.gcr.io/pause:3.1") + require.Equal(t, "registry.io/pause:3.1", out) + + out = getImageURN("", "registry.io//", "gcr.io/pause:3.1") + require.Equal(t, "registry.io/pause:3.1", out) + + out = getImageURN("gcr.io,k8s.gcr.io", "registry.io", "gcr.io/pause:3.1") + require.Equal(t, "registry.io/pause:3.1", out) + + out = getImageURN("gcr.io,k8s.gcr.io", "registry.io", "k8s.gcr.io/pause:3.1") + require.Equal(t, "registry.io/pause:3.1", out) + + out = getImageURN("gcr.io,k8s.gcr.io", "registry.io", "testrepo/pause:3.1") + require.Equal(t, "registry.io/testrepo/pause:3.1", out) +} + +func getImageURN(commonRegistries string, customImageRegistry string, image string) string { + cluster := corev1.StorageCluster{} + cluster.Annotations = make(map[string]string) + cluster.Annotations[constants.AnnotationCommonImageRegistries] = commonRegistries + cluster.Spec.CustomImageRegistry = customImageRegistry + return GetImageURN(&cluster, image) } func TestGetImageMajorVersion(t *testing.T) {