diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index fd408beee0ed..c0e5c6a67b27 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -72,6 +72,9 @@ func AllTests() []upgrades.Test { &prometheus.MetricsAvailableAfterUpgradeTest{}, &dns.UpgradeTest{}, &router.GatewayAPIUpgradeTest{}, + &router.HAProxyVersionUpgradeTest{Mode: router.HAProxyUpgradeModeUnset}, + &router.HAProxyVersionUpgradeTest{Mode: router.HAProxyUpgradeModeNonDefault}, + &router.HAProxyVersionUpgradeTest{Mode: router.HAProxyUpgradeModeDefault}, } } diff --git a/test/extended/router/haproxyversion_upgrade.go b/test/extended/router/haproxyversion_upgrade.go new file mode 100644 index 000000000000..e8c94b049ebb --- /dev/null +++ b/test/extended/router/haproxyversion_upgrade.go @@ -0,0 +1,293 @@ +package router + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/upgrades" + + operatorv1 "github.com/openshift/api/operator/v1" + operatorv1client "github.com/openshift/client-go/operator/clientset/versioned" + exutil "github.com/openshift/origin/test/extended/util" +) + +// HAProxyVersionUpgradeTest verifies that HAProxy version selection behaves +// as expected during upgrades. +// Mode is a test parameter that should define how the HAProxy version +// should be configured before the upgrade. +type HAProxyVersionUpgradeTest struct { + Mode HAProxyUpgradeMode + + // internal state + oc *exutil.CLI + operatorClient operatorv1client.Interface + controllers *ingressControllers + versionConfig haproxyVersionConfig + pinnedVersion operatorv1.HAProxyVersion + precheckErr error + ic types.NamespacedName +} + +// HAProxyUpgradeMode is the mode of the HAProxy upgrade. +type HAProxyUpgradeMode string + +const ( + // HAProxyUpgradeModeUnset defines the HAProxy version as unpinned before the upgrade. + HAProxyUpgradeModeUnset HAProxyUpgradeMode = "unset" + // HAProxyUpgradeModeDefault defines the HAProxy version with the default version before the upgrade. + HAProxyUpgradeModeDefault HAProxyUpgradeMode = "default" + // HAProxyUpgradeModeNonDefault defines the HAProxy version with a non default but supported version before the upgrade. + HAProxyUpgradeModeNonDefault HAProxyUpgradeMode = "non-default" +) + +func (h *HAProxyVersionUpgradeTest) Name() string { + return "haproxy-version-upgrade-" + string(h.Mode) +} + +func (h *HAProxyVersionUpgradeTest) DisplayName() string { + return fmt.Sprintf("[sig-network-edge][Feature:Router][apigroup:route.openshift.io] Verify HAProxy %s version state during upgrade", h.Mode) +} + +// Skip returns true when the test cannot safely run: the API lacks the haproxyVersion field, the upgrade +// is a multi-hop chain with a pinned version, or (for NonDefault) no safe non-default version is available. +func (h *HAProxyVersionUpgradeTest) Skip(upgctx upgrades.UpgradeContext) bool { + framework.Logf("Upgrade config: %+v", upgctx) + + if h.Mode != HAProxyUpgradeModeUnset && len(upgctx.Versions) > 2 { + // We could have a deprecation and dropping version in the middle of a + // multi-hop upgrade, so we cannot safely run the test having HAProxy pinned. + framework.Logf("skipping: cannot test a multi-hop upgrade with HAProxy pinned. mode=%q, versions=%d", h.Mode, len(upgctx.Versions)) + return true + } + + ctx := context.Background() + + oc := exutil.NewCLIForMonitorTest(h.Name() + "-skip").AsAdmin() + hasField, err := apiHasHAProxyVersionField(ctx, oc) + if err != nil { + h.precheckErr = fmt.Errorf("error checking for HAProxy version API: %w", err) + return false + } + if !hasField { + framework.Logf("skipping: IngressController API is missing the haproxyVersion field") + return true + } + + versions, err := getHAProxyVersionConfig(ctx, oc) + if err != nil { + h.precheckErr = fmt.Errorf("error getting HAProxy version config: %w", err) + return false + } + framework.Logf("HAProxy version config: %+v", versions) + + if h.Mode == HAProxyUpgradeModeNonDefault && len(versions.getNonDefaultVersions()) == 0 { + framework.Logf("skipping: cannot use non default: there are no non default versions") + return true + } + + if h.Mode == HAProxyUpgradeModeNonDefault && len(versions.getNonDefaultUpgradeableVersions()) == 0 { + // Strictly, only a y-stream upgrade could drop this version, but Skip() cannot + // reliably tell y-stream from z-stream before the upgrade completes (the target + // may be given as a pull-spec, not a parseable Version), so we skip conservatively + // regardless of upgrade type. + framework.Logf("skipping: cannot use non default: the only available non default version is deprecated") + return true + } + + h.versionConfig = versions + h.precheckErr = nil + return false +} + +// Setup configures all the test attributes and creates an IngressController +// resource that should be verified after the upgrade. +func (h *HAProxyVersionUpgradeTest) Setup(ctx context.Context, f *framework.Framework) { + o.Expect(h.precheckErr).NotTo(o.HaveOccurred(), "Skip() precheck failed: could not determine if HAProxy version upgrade test should run") + + g.By("Setting up HAProxy version test") + + h.oc = exutil.NewCLIWithFramework(f).AsAdmin() + h.operatorClient = h.oc.AdminOperatorClient() + h.controllers = &ingressControllers{} + + var haproxyVersion operatorv1.HAProxyVersion + switch h.Mode { + case HAProxyUpgradeModeUnset: + haproxyVersion = "" + case HAProxyUpgradeModeDefault: + haproxyVersion = h.versionConfig.defaultVersion + case HAProxyUpgradeModeNonDefault: + haproxyVersion = h.versionConfig.getNonDefaultUpgradeableVersions()[0] + default: + framework.Failf("unsupported test mode: %q", h.Mode) + } + + g.By("Creating the IngressController resource") + + ic, err := h.controllers.createIngressController(ctx, h.oc, func(controller *operatorv1.IngressController) { + controller.Spec.HAProxyVersion = haproxyVersion + }) + o.Expect(err).NotTo(o.HaveOccurred(), "error creating IngressController resource") + h.ic = types.NamespacedName{ + Namespace: ic.Namespace, + Name: ic.Name, + } + h.pinnedVersion = haproxyVersion + + framework.Logf("Created IngressController %s with spec.haproxyVersion=%q", h.ic.String(), haproxyVersion) + + g.By("Checking HAProxy version for Ingress " + h.ic.String()) + + waitingVersion := haproxyVersion + if waitingVersion == "" { + waitingVersion = h.versionConfig.defaultVersion + } + err = waitForHAProxyVersion(ctx, h.oc, ic.Name, waitingVersion) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy version from runtime API") +} + +// Test verifies that the expected HAProxy version is found after the upgrade. +// Current version is read from the IngressController status and from the +// HAProxy's runtime API. +func (h *HAProxyVersionUpgradeTest) Test(ctx context.Context, f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) { + g.By("Waiting for upgrade to complete") + <-done + + err := waitForIngressControllerReady(h.oc, h.ic) + o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("error waiting for IngressController %s to be ready", h.ic.String())) + + g.By("Validating HAProxy version after upgrade") + + versions, err := getHAProxyVersionConfig(ctx, h.oc) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy version config") + framework.Logf("HAProxy version config: %+v", versions) + + var expectedVersion operatorv1.HAProxyVersion + switch h.Mode { + case HAProxyUpgradeModeUnset: + expectedVersion = versions.defaultVersion + case HAProxyUpgradeModeDefault, HAProxyUpgradeModeNonDefault: + expectedVersion = h.pinnedVersion + default: + framework.Failf("unsupported test mode: %q", h.Mode) + } + + framework.Logf("Post-upgrade HAProxy version check: expected=%s", expectedVersion) + + const rollingOutTimeout = 15 * time.Minute + err = waitForEffectiveHAProxyVersion(ctx, h.operatorClient, h.ic, expectedVersion, rollingOutTimeout) + o.Expect(err).NotTo(o.HaveOccurred(), "error waiting for EffectiveHAProxyVersion") + + g.By("Validating HAProxy version from runtime API") + + err = waitForHAProxyVersion(ctx, h.oc, h.ic.Name, expectedVersion) + o.Expect(err).NotTo(o.HaveOccurred(), "error getting HAProxy version from runtime API") +} + +// Teardown removes the configured IngressController after the test runs. +func (h *HAProxyVersionUpgradeTest) Teardown(ctx context.Context, f *framework.Framework) { + if h.operatorClient == nil { + framework.Logf("Skipping cleanup because setup did not initialize test resources") + return + } + if err := h.controllers.deleteAll(ctx, h.operatorClient); err != nil { + framework.Logf("error deleting IngressController resource: %s", err.Error()) + } +} + +// haproxyVersionConfig has HAProxy version configuration from the Ingress operator. +type haproxyVersionConfig struct { + defaultVersion operatorv1.HAProxyVersion + deprecatedVersion operatorv1.HAProxyVersion + availableVersions []operatorv1.HAProxyVersion +} + +// getHAProxyVersionConfig parses the current Ingress operator configuration +// and extracts the HAProxy version configuration. +func getHAProxyVersionConfig(ctx context.Context, oc *exutil.CLI) (haproxyVersionConfig, error) { + operatorNamespace := "openshift-ingress-operator" + operatorName := "ingress-operator" + deploy, err := oc.AdminKubeClient().AppsV1().Deployments(operatorNamespace).Get(ctx, operatorName, metav1.GetOptions{}) + if err != nil { + return haproxyVersionConfig{}, err + } + + containers := deploy.Spec.Template.Spec.Containers + if len(containers) < 1 { + return haproxyVersionConfig{}, fmt.Errorf("ingress-operator deployment is missing the operator container") + } + + operator := containers[0] + if operator.Name != operatorName { + return haproxyVersionConfig{}, fmt.Errorf("ingress-operator deployment has an unexpected container name: %s", operator.Name) + } + + // Read default and deprecated versions from Env + var defaultVersion, deprecatedVersion operatorv1.HAProxyVersion + for _, env := range operator.Env { + switch env.Name { + case "DEFAULT_HAPROXY_VERSION": + defaultVersion = operatorv1.HAProxyVersion(env.Value) + case "DEPRECATED_HAPROXY_VERSION": + deprecatedVersion = operatorv1.HAProxyVersion(env.Value) + } + } + if defaultVersion == "" { + // envvar not found, so this is pre 4.23/5.0, assume "2.8" + defaultVersion = "2.8" + } + + // Read available versions from Command. + // The available versions are configured this way: + // + // command: + // - ... + // - --haproxy-image + // - "2.8=$(HAPROXY_28_IMAGE)" + // - --haproxy-image + // - "3.2=$(HAPROXY_32_IMAGE)" + // + var availableVersions []operatorv1.HAProxyVersion + cmds := operator.Command + for i := range cmds { + if cmds[i] == "--haproxy-image" && len(cmds) > i+1 { + // "2.8=$(HAPROXY_28_IMAGE)" + value := cmds[i+1] + // ["2.8", "$(HAPROXY_28_IMAGE)"] + version := strings.Split(value, "=") + availableVersions = append(availableVersions, operatorv1.HAProxyVersion(version[0])) + } + } + if len(availableVersions) == 0 { + // --haproxy-image not configured, so this is pre 4.23/5.0, assume [defaultVersion] + availableVersions = []operatorv1.HAProxyVersion{defaultVersion} + } + + return haproxyVersionConfig{ + defaultVersion: defaultVersion, + deprecatedVersion: deprecatedVersion, + availableVersions: availableVersions, + }, nil +} + +// getNonDefaultVersions creates a list of non default versions, derived from the default and the available ones. +func (h *haproxyVersionConfig) getNonDefaultVersions() []operatorv1.HAProxyVersion { + return slices.DeleteFunc(slices.Clone(h.availableVersions), func(v operatorv1.HAProxyVersion) bool { + return v == h.defaultVersion + }) +} + +// getNonDefaultUpgradeableVersions creates a list of non default and upgradeable versions, derived from the default, the deprecated, and the available ones. +func (h *haproxyVersionConfig) getNonDefaultUpgradeableVersions() []operatorv1.HAProxyVersion { + return slices.DeleteFunc(slices.Clone(h.availableVersions), func(v operatorv1.HAProxyVersion) bool { + return v == h.defaultVersion || v == h.deprecatedVersion + }) +} diff --git a/test/extended/router/multi-haproxy.go b/test/extended/router/multi-haproxy.go index 7536293414a2..be013417ba1e 100644 --- a/test/extended/router/multi-haproxy.go +++ b/test/extended/router/multi-haproxy.go @@ -5,25 +5,28 @@ import ( "errors" "fmt" "strings" + "sync" "time" g "github.com/onsi/ginkgo/v2" o "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/storage/names" + e2e "k8s.io/kubernetes/test/e2e/framework" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" operatorv1 "github.com/openshift/api/operator/v1" - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + operatorv1client "github.com/openshift/client-go/operator/clientset/versioned" "github.com/openshift/origin/test/extended/router/shard" exutil "github.com/openshift/origin/test/extended/util" - e2e "k8s.io/kubernetes/test/e2e/framework" ) var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift.io][OCPFeatureGate:IngressControllerMultipleHAProxyVersions]", func() { @@ -32,10 +35,10 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. // testsTimeout defines the maximum amount of time to wait for test operations to complete. const testsTimeout = 5 * time.Minute - // defaultHAProxyVersion is the default HAProxy version for the current release. - var defaultHAProxyVersion operatorv1.HAProxyVersion + // versionConfig is the HAProxy version configuration in the current release. + var versionConfig haproxyVersionConfig - //alternateHAProxyVersion is the other accepted HAProxyVersion accepted in the current release + // alternateHAProxyVersion is one of the non default accepted HAProxyVersions in the current release. var alternateHAProxyVersion operatorv1.HAProxyVersion // controllers is used to create new ingress controllers, and stores their reference so they can be removed after the test runs @@ -51,46 +54,29 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. exutil.DumpPodLogsStartingWithInNamespace(ic.controller.Name, ic.controller.Namespace, oc) } } - var errs []error - for _, ic := range controllers.items { - err := operatorClient.OperatorV1().IngressControllers(ic.controller.Namespace).Delete(ctx, ic.controller.Name, *metav1.NewDeleteOptions(1)) - errs = append(errs, client.IgnoreNotFound(err)) - } - o.Expect(errors.Join(errs...)).NotTo(o.HaveOccurred()) + err := controllers.deleteAll(ctx, operatorClient) + o.Expect(err).NotTo(o.HaveOccurred()) controllers.items = nil }) g.BeforeEach(func() { - - apiExtClient, err := apiextensionsclient.NewForConfig(oc.AdminConfig()) - o.Expect(err).NotTo(o.HaveOccurred()) - - crd, err := apiExtClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "ingresscontrollers.operator.openshift.io", metav1.GetOptions{}) + hasField, err := apiHasHAProxyVersionField(ctx, oc) o.Expect(err).NotTo(o.HaveOccurred()) - - // Check if haproxyVersion field exists in the CRD schema - hasField := false - for _, v := range crd.Spec.Versions { - if v.Name == "v1" && v.Schema != nil && v.Schema.OpenAPIV3Schema != nil { - if _, ok := v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["haproxyVersion"]; ok { - hasField = true - } - } - } if !hasField { g.Skip("IngressController CRD does not have haproxyVersion field — operator not yet updated") } - defaultIC, err := operatorClient.OperatorV1().IngressControllers("openshift-ingress-operator").Get(ctx, "default", metav1.GetOptions{}) + versions, err := getHAProxyVersionConfig(ctx, oc) o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(defaultIC.Status.EffectiveHAProxyVersion).NotTo(o.BeEmpty()) - defaultHAProxyVersion = defaultIC.Status.EffectiveHAProxyVersion - if defaultHAProxyVersion == operatorv1.HAProxyVersion28 { - alternateHAProxyVersion = operatorv1.HAProxyVersion32 - } else { - alternateHAProxyVersion = operatorv1.HAProxyVersion28 + nonDefaultVersions := versions.getNonDefaultVersions() + if len(nonDefaultVersions) == 0 { + g.Skip("IngressController has no non default versions available") } + + // update shared vars + versionConfig = versions + alternateHAProxyVersion = nonDefaultVersions[0] }) g.Describe("The HAProxy router with version selection", func() { @@ -98,15 +84,15 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. // Ensure that the haproxyVersion field in the IngressController API does not accept unknown versions g.It("should reject invalid HAProxy versions", func() { versions := []operatorv1.HAProxyVersion{ - " ", // Empty becomes unset, but space is invalid - "2.6", // one LTS before the oldest supported version, so always invalid - "v" + defaultHAProxyVersion, // v prefix is invalid - defaultHAProxyVersion + ".0", // .z suffix is invalid, only x.y is supported - " " + defaultHAProxyVersion, // leading space is invalid - defaultHAProxyVersion + " ", // trailing space is invalid + " ", // Empty becomes unset, but space is invalid + "2.6", // one LTS before the oldest supported version, so always invalid + "v" + versionConfig.defaultVersion, // v prefix is invalid + versionConfig.defaultVersion + ".0", // .z suffix is invalid, only x.y is supported + " " + versionConfig.defaultVersion, // leading space is invalid + versionConfig.defaultVersion + " ", // trailing space is invalid } for _, version := range versions { - _, err := controllers.createIngressController(ctx, oc, testsTimeout, func(ic *operatorv1.IngressController) { + _, err := controllers.createIngressController(ctx, oc, func(ic *operatorv1.IngressController) { ic.Spec.HAProxyVersion = version }) o.Expect(err).To(o.Not(o.Succeed())) @@ -116,7 +102,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. // Ensure that the ingress controller reverts back to the default version after unsetting the field with null g.It("should revert to default HAProxy version when field is cleared", func() { - ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, func(ic *operatorv1.IngressController) { + ingress, err := controllers.createIngressController(ctx, oc, func(ic *operatorv1.IngressController) { ic.Spec.HAProxyVersion = alternateHAProxyVersion }) o.Expect(err).NotTo(o.HaveOccurred()) @@ -131,34 +117,18 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. o.Expect(err).NotTo(o.HaveOccurred()) g.By("Confirm that the HAProxy version shows default version") - err = waitForHAProxyVersion(ctx, oc, ingress.Name, defaultHAProxyVersion) + err = waitForHAProxyVersion(ctx, oc, ingress.Name, versionConfig.defaultVersion) o.Expect(err).NotTo(o.HaveOccurred()) }) // Ensure that the running HAProxy version matches the version configured in the IngressController API g.It("should configure the same HAProxy version defined in the API", func() { - versions := []operatorv1.HAProxyVersion{ - defaultHAProxyVersion, - alternateHAProxyVersion, - } - for _, version := range versions { - ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, func(ic *operatorv1.IngressController) { + for _, version := range versionConfig.availableVersions { + ingress, err := controllers.createIngressController(ctx, oc, func(ic *operatorv1.IngressController) { ic.Spec.HAProxyVersion = version }) o.Expect(err).To(o.Succeed()) - errPoll := wait.PollUntilContextTimeout(ctx, 2*time.Second, testsTimeout, true, func(ctx context.Context) (bool, error) { - ic, err := operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Get(ctx, ingress.Name, metav1.GetOptions{}) - if err != nil { - e2e.Logf("Failed to get the IngressController %s", ingress.Name) - return false, nil - } - if ic.Status.EffectiveHAProxyVersion == version { - e2e.Logf("EffectiveHAProxyVersion shows the expected version: %q", version) - return true, nil - } - e2e.Logf("EffectiveHAProxyVersion: %q does not match the expected version %q", ic.Status.EffectiveHAProxyVersion, version) - return false, nil - }) + errPoll := waitForEffectiveHAProxyVersion(ctx, operatorClient, types.NamespacedName{Namespace: ingress.Namespace, Name: ingress.Name}, version, testsTimeout) o.Expect(errPoll).NotTo(o.HaveOccurred(), "Timed out waiting for EffectiveHAProxyVersion") e2e.Logf("IngressController: %s matches the expected HAProxyVersion: %s", ingress.Name, string(version)) } @@ -167,30 +137,16 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. // Ensure that if the version is unset, its value in the IngressController API remains undeclared, and the running HAProxy matches the default version g.It("should configure the default HAProxy if the version is unset", func() { // create a custom ingress controller with an unset HAProxyVersion - ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, nil) + ingress, err := controllers.createIngressController(ctx, oc, nil) o.Expect(err).To(o.Succeed()) //confirm that the ingresscontroller is unset o.Expect(ingress.Spec.HAProxyVersion).To(o.BeEmpty()) - var effectiveVersion operatorv1.HAProxyVersion - err = wait.PollUntilContextTimeout(ctx, 2*time.Second, testsTimeout, true, func(ctx context.Context) (bool, error) { - ic, err := operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Get(ctx, ingress.Name, metav1.GetOptions{}) - if err != nil { - e2e.Logf("Failed to get the IngressController %s", ingress.Name) - return false, nil - } - if ic.Status.EffectiveHAProxyVersion == "" { - e2e.Logf("IngressController %s: EffectiveHAProxyVersion not yet set, waiting...", ingress.Name) - return false, nil - } - effectiveVersion = ic.Status.EffectiveHAProxyVersion - return true, nil - }) + err = waitForEffectiveHAProxyVersion(ctx, operatorClient, types.NamespacedName{Namespace: ingress.Namespace, Name: ingress.Name}, versionConfig.defaultVersion, testsTimeout) o.Expect(err).NotTo(o.HaveOccurred(), "Timed out waiting for EffectiveHAProxyVersion") - o.Expect(effectiveVersion).To(o.Equal(defaultHAProxyVersion)) - e2e.Logf("IngressController: %s has the expected HAProxyVersion: %s", ingress.Name, string(defaultHAProxyVersion)) - err = waitForHAProxyVersion(ctx, oc, ingress.Name, defaultHAProxyVersion) + e2e.Logf("IngressController: %s has the expected HAProxyVersion: %s", ingress.Name, string(versionConfig.defaultVersion)) + err = waitForHAProxyVersion(ctx, oc, ingress.Name, versionConfig.defaultVersion) o.Expect(err).NotTo(o.HaveOccurred()) }) @@ -202,7 +158,7 @@ var _ = g.Describe("[sig-network-edge][Feature:Router][apigroup:route.openshift. ingressVersion := ingressDefault.Status.EffectiveHAProxyVersion g.By("Create a custom controller and patch it to an older version") - ingress, err := controllers.createIngressController(ctx, oc, testsTimeout, nil) + ingress, err := controllers.createIngressController(ctx, oc, nil) o.Expect(err).To(o.Succeed()) patch := []byte(fmt.Sprintf(`{"spec":{"haproxyVersion":"%s"}}`, alternateHAProxyVersion)) @@ -229,7 +185,7 @@ type ingressController struct { controller types.NamespacedName } -func (i *ingressControllers) createIngressController(ctx context.Context, oc *exutil.CLI, readyTimeout time.Duration, custom func(ic *operatorv1.IngressController)) (*operatorv1.IngressController, error) { +func (i *ingressControllers) createIngressController(ctx context.Context, oc *exutil.CLI, custom func(ic *operatorv1.IngressController)) (*operatorv1.IngressController, error) { operatorClient := oc.AdminOperatorClient() // ingress controller need to be created in operator's namespace, ... @@ -272,20 +228,62 @@ func (i *ingressControllers) createIngressController(ctx context.Context, oc *ex } i.items = append(i.items, &ictr) + return ingress, waitForIngressControllerReady(oc, controller) +} + +func (i *ingressControllers) deleteAll(ctx context.Context, operatorClient operatorv1client.Interface) error { + errs := make([]error, len(i.items)) + wg := sync.WaitGroup{} + for idx, ic := range i.items { + wg.Go(func() { + if err := deleteIngressControllerAndWait(ctx, operatorClient, ic.controller); err != nil { + errs[idx] = fmt.Errorf("error during IngressController %s deletion: %w", ic.controller.String(), err) + } + }) + } + wg.Wait() + return errors.Join(errs...) +} + +// waitForIngressControllerReady waits for the provided IngressController to be ready. +func waitForIngressControllerReady(oc *exutil.CLI, ic types.NamespacedName) error { ingressControllerReady := []operatorv1.OperatorCondition{ {Type: operatorv1.IngressControllerAvailableConditionType, Status: operatorv1.ConditionTrue}, {Type: operatorv1.LoadBalancerManagedIngressConditionType, Status: operatorv1.ConditionFalse}, {Type: operatorv1.DNSManagedIngressConditionType, Status: operatorv1.ConditionFalse}, {Type: operatorv1.OperatorStatusTypeProgressing, Status: operatorv1.ConditionFalse}, } + return shard.WaitForIngressControllerCondition(oc, 5*time.Minute, ic, ingressControllerReady...) +} - // wait for the controller to be available - err = shard.WaitForIngressControllerCondition(oc, readyTimeout, controller, ingressControllerReady...) - if err != nil { - return nil, err - } +// waitForIngressControllerDeletion waits for an IngressController to be removed. +func waitForIngressControllerDeletion(ctx context.Context, operatorClient operatorv1client.Interface, ic types.NamespacedName) error { + return wait.PollUntilContextTimeout(ctx, 2*time.Second, 5*time.Minute, false, func(ctx context.Context) (done bool, err error) { + _, err = operatorClient.OperatorV1().IngressControllers(ic.Namespace).Get(ctx, ic.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + e2e.Logf("IngressController %s has been deleted", ic.String()) + return true, nil + } + if err != nil { + e2e.Logf("error reading IngressController %s: %s", ic.String(), err.Error()) + } else { + e2e.Logf("waiting IngressController %s to be deleted", ic.String()) + } + return false, nil + }) +} - return ingress, nil +// deleteIngressControllerAndWait deletes an IngressController and waits for it to be removed. +func deleteIngressControllerAndWait(ctx context.Context, operatorClient operatorv1client.Interface, ic types.NamespacedName) error { + e2e.Logf("Deleting IngressController %s", ic.String()) + err := operatorClient.OperatorV1().IngressControllers(ic.Namespace).Delete(ctx, ic.Name, *metav1.NewDeleteOptions(1)) + if client.IgnoreNotFound(err) != nil { + return fmt.Errorf("error deleting IngressController %s: %w", ic.String(), err) + } + if err := waitForIngressControllerDeletion(ctx, operatorClient, ic); err != nil { + return fmt.Errorf("IngressController %s was not deleted: %w", ic.String(), err) + } + return nil } // poll the router pods HAProxy Container to check that the version is correctly asserted @@ -308,3 +306,45 @@ func waitForHAProxyVersion(ctx context.Context, oc *exutil.CLI, ingressName stri }) return err } + +// waitForEffectiveHAProxyVersion polls the provided IngressController until its EffectiveHAProxyVersion equals expectedVersion. +func waitForEffectiveHAProxyVersion(ctx context.Context, operatorClient operatorv1client.Interface, ingress types.NamespacedName, expectedVersion operatorv1.HAProxyVersion, testsTimeout time.Duration) error { + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, testsTimeout, true, func(ctx context.Context) (bool, error) { + ic, err := operatorClient.OperatorV1().IngressControllers(ingress.Namespace).Get(ctx, ingress.Name, metav1.GetOptions{}) + if err != nil { + e2e.Logf("Failed to get the IngressController %s: %s", ingress.String(), err.Error()) + return false, nil + } + if ic.Status.EffectiveHAProxyVersion != expectedVersion { + e2e.Logf("IngressController %s: HAProxy version %q does not match expected value %q", ingress.String(), ic.Status.EffectiveHAProxyVersion, expectedVersion) + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("error waiting for EffectiveHAProxyVersion to match expected version: %s", err.Error()) + } + return nil +} + +func apiHasHAProxyVersionField(ctx context.Context, oc *exutil.CLI) (bool, error) { + apiExtClient, err := apiextensionsclient.NewForConfig(oc.AdminConfig()) + if err != nil { + return false, err + } + + crd, err := apiExtClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "ingresscontrollers.operator.openshift.io", metav1.GetOptions{}) + if err != nil { + return false, err + } + + // Check if haproxyVersion field exists in the CRD schema + for _, v := range crd.Spec.Versions { + if v.Name == "v1" && v.Schema != nil && v.Schema.OpenAPIV3Schema != nil { + if _, ok := v.Schema.OpenAPIV3Schema.Properties["spec"].Properties["haproxyVersion"]; ok { + return true, nil + } + } + } + return false, nil +}