diff --git a/test/extended/networking/egressip.go b/test/extended/networking/egressip.go index 3bbb9fd67964..0ce345e687a3 100644 --- a/test/extended/networking/egressip.go +++ b/test/extended/networking/egressip.go @@ -5,7 +5,9 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net" "os" + "regexp" "strings" "time" @@ -85,486 +87,735 @@ var _ = g.Describe("[sig-network][Feature:EgressIP][apigroup:operator.openshift. o.Expect(err).NotTo(o.HaveOccurred()) g.By("Getting the kubernetes clientset") - f := oc.KubeFramework() - clientset = f.ClientSet - - g.By("Getting the cloudnetwork clientset") - cloudNetworkClientset, err = cloudnetwork.NewForConfig(oc.AdminConfig()) - o.Expect(err).NotTo(o.HaveOccurred()) - - g.By("Determining the cloud infrastructure type") - infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) - o.Expect(err).NotTo(o.HaveOccurred()) - cloudType = infra.Spec.PlatformSpec.Type - - g.By("Verifying that this is a supported cloud infrastructure platform") - isSupportedPlatform := false - supportedPlatforms := []configv1.PlatformType{ - configv1.AWSPlatformType, - configv1.GCPPlatformType, - configv1.AzurePlatformType, - configv1.OpenStackPlatformType, - } - for _, supportedPlatform := range supportedPlatforms { - if cloudType == supportedPlatform { - isSupportedPlatform = true - break - } - } - if !isSupportedPlatform { - skipper.Skipf("This cloud platform (%s) is not supported for this test", cloudType) - } - - // A supported version of OpenShift must hold the CloudPrivateIPConfig CRD. - // Otherwise, skip this test. - g.By("Verifying that this is a supported version of OpenShift") - isSupportedOcpVersion, err := exutil.DoesApiResourceExist(oc.AdminConfig(), "cloudprivateipconfigs", "cloud.network.openshift.io") - o.Expect(err).NotTo(o.HaveOccurred()) - if !isSupportedOcpVersion { - skipper.Skipf("This OCP version is not supported for this test (api-resource cloudprivateipconfigs not found)") - } + clientset = oc.KubeFramework().ClientSet g.By("Getting all worker nodes in alphabetical order") - // Get all worker nodes, order them alphabetically with stable - // sort order. workerNodesOrdered, err = getWorkerNodesOrdered(clientset) o.Expect(err).NotTo(o.HaveOccurred()) + workerNodesOrderedNames = nil for _, s := range workerNodesOrdered { workerNodesOrderedNames = append(workerNodesOrderedNames, s.Name) } if len(workerNodesOrdered) < 3 { skipper.Skipf("This test requires a minimum of 3 worker nodes. However, this environment has %d worker nodes.", len(workerNodesOrdered)) } + }) - g.By("Determining the cloud address families") - hasIPv4, hasIPv6, err = GetIPAddressFamily(oc) - o.Expect(err).NotTo(o.HaveOccurred()) + g.AfterEach(func() { + g.By("Removing the temp directory") + os.RemoveAll(tmpDirEgressIP) + }) - g.By("Determining the target protocol, host and port") - targetProtocol, targetHost, targetPort, err = getTargetProtocolHostPort(oc, hasIPv4, hasIPv6, cloudType) - o.Expect(err).NotTo(o.HaveOccurred()) - framework.Logf("Testing against: CloudType: %s, Protocol %s, TargetHost: %s, TargetPort: %d", - cloudType, - targetProtocol, - targetHost, - targetPort) - - g.By("Creating a project for the prober pod") - // Create a target project and assign source and target namespace - // to variables for later use. - egressIPNamespace = f.Namespace.Name - externalNamespace = oc.SetupProject() - - g.By("Selecting the EgressIP nodes and a non-EgressIP node") - nonEgressIPNodeName = workerNodesOrderedNames[0] - egressIPNodesOrderedNames = workerNodesOrderedNames[1:] - - g.By("Setting the ingressdomain") - ingressDomain, err = getIngressDomain(oc) - o.Expect(err).NotTo(o.HaveOccurred()) + g.Context("cloud platform tests", func() { + g.BeforeEach(func() { + var err error - g.By("Setting the EgressIP nodes as EgressIP assignable") - for _, node := range egressIPNodesOrderedNames { - _, err = runOcWithRetry(oc.AsAdmin(), "label", "node", node, "k8s.ovn.org/egress-assignable=") + g.By("Getting the cloudnetwork clientset") + cloudNetworkClientset, err = cloudnetwork.NewForConfig(oc.AdminConfig()) o.Expect(err).NotTo(o.HaveOccurred()) - } - }) - // Do not check for errors in g.AfterEach as the other cleanup steps will fail, otherwise. - g.AfterEach(func() { - g.By("Deleting the EgressIP object if it exists") - egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml - if _, err := os.Stat(egressIPYamlPath); err == nil { - _, _ = runOcWithRetry(oc.AsAdmin(), "delete", "-f", tmpDirEgressIP+"/"+egressIPYaml) - } + g.By("Determining the cloud infrastructure type") + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + cloudType = infra.Spec.PlatformSpec.Type + + g.By("Verifying that this is a supported cloud infrastructure platform") + isSupportedPlatform := false + supportedPlatforms := []configv1.PlatformType{ + configv1.AWSPlatformType, + configv1.GCPPlatformType, + configv1.AzurePlatformType, + configv1.OpenStackPlatformType, + } + for _, supportedPlatform := range supportedPlatforms { + if cloudType == supportedPlatform { + isSupportedPlatform = true + break + } + } + if !isSupportedPlatform { + skipper.Skipf("This cloud platform (%s) is not supported for this test", cloudType) + } - g.By("Removing the EgressIP assignable annotation") - for _, nodeName := range egressIPNodesOrderedNames { - _, _ = runOcWithRetry(oc.AsAdmin(), "label", "node", nodeName, "k8s.ovn.org/egress-assignable-") - } + // A supported version of OpenShift must hold the CloudPrivateIPConfig CRD. + // Otherwise, skip this test. + g.By("Verifying that this is a supported version of OpenShift") + isSupportedOcpVersion, err := exutil.DoesApiResourceExist(oc.AdminConfig(), "cloudprivateipconfigs", "cloud.network.openshift.io") + o.Expect(err).NotTo(o.HaveOccurred()) + if !isSupportedOcpVersion { + skipper.Skipf("This OCP version is not supported for this test (api-resource cloudprivateipconfigs not found)") + } - g.By("Removing the temp directory") - os.RemoveAll(tmpDirEgressIP) - }) + g.By("Determining the cloud address families") + hasIPv4, hasIPv6, err = GetIPAddressFamily(oc) + o.Expect(err).NotTo(o.HaveOccurred()) - g.Context("[internal-targets]", func() { - g.JustBeforeEach(func() { - // Host networked is needed for host networked pods. - g.By("Adding SCC hostnetwork to the external namespace") - _, err := runOcWithRetry(oc.AsAdmin(), "adm", "policy", "add-scc-to-user", "hostnetwork", fmt.Sprintf("system:serviceaccount:%s:default", externalNamespace)) + g.By("Determining the target protocol, host and port") + targetProtocol, targetHost, targetPort, err = getTargetProtocolHostPort(oc, hasIPv4, hasIPv6, cloudType) o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Testing against: CloudType: %s, Protocol %s, TargetHost: %s, TargetPort: %d", + cloudType, + targetProtocol, + targetHost, + targetPort) + + g.By("Creating a project for the prober pod") + f := oc.KubeFramework() + egressIPNamespace = f.Namespace.Name + externalNamespace = oc.SetupProject() + + g.By("Selecting the EgressIP nodes and a non-EgressIP node") + nonEgressIPNodeName = workerNodesOrderedNames[0] + egressIPNodesOrderedNames = workerNodesOrderedNames[1:] + + g.By("Setting the ingressdomain") + ingressDomain, err = getIngressDomain(oc) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Setting the EgressIP nodes as EgressIP assignable") + for _, node := range egressIPNodesOrderedNames { + _, err = runOcWithRetry(oc.AsAdmin(), "label", "node", node, "k8s.ovn.org/egress-assignable=") + o.Expect(err).NotTo(o.HaveOccurred()) + } }) - g.It("EgressIP pods should query hostNetwork pods with the local node's SNAT", func() { - var targetIP string - var targetPort int - - g.By("Selecting a single EgressIP node, and one node per source deployment") - // Requires a minimum of 3 worker nodes in total: - // 1 nonEgressIPNodeName + at least 2 as sources of EgressIP traffic. - o.Expect(len(egressIPNodesOrderedNames)).Should(o.BeNumerically(">", 1)) - egressIPNodeStr := []string{egressIPNodesOrderedNames[0]} - deploymentNodeStr := [][]string{ - {egressIPNodesOrderedNames[0]}, - {egressIPNodesOrderedNames[1]}, + // Do not check for errors in g.AfterEach as the other cleanup steps will fail, otherwise. + g.AfterEach(func() { + g.By("Deleting the EgressIP object if it exists") + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + if _, err := os.Stat(egressIPYamlPath); err == nil { + _, _ = runOcWithRetry(oc.AsAdmin(), "delete", "-f", tmpDirEgressIP+"/"+egressIPYaml) + } + + g.By("Removing the EgressIP assignable annotation") + for _, nodeName := range egressIPNodesOrderedNames { + _, _ = runOcWithRetry(oc.AsAdmin(), "label", "node", nodeName, "k8s.ovn.org/egress-assignable-") } + }) - g.By("Creating the target DaemonSet with a single hostnetworked pod on the target node") - daemonSetName := "hostnetworked" - // Try the entire port range to create the DaemonSet. - for i := 0; i < egressIPTargetHostPortMax-egressIPTargetHostPortMin; i++ { - containerPort, err := portAllocator.AllocateNextPort() + g.Context("[internal-targets]", func() { + g.JustBeforeEach(func() { + // Host networked is needed for host networked pods. + g.By("Adding SCC hostnetwork to the external namespace") + _, err := runOcWithRetry(oc.AsAdmin(), "adm", "policy", "add-scc-to-user", "hostnetwork", fmt.Sprintf("system:serviceaccount:%s:default", externalNamespace)) o.Expect(err).NotTo(o.HaveOccurred()) + }) + + g.It("EgressIP pods should query hostNetwork pods with the local node's SNAT", func() { + var targetIP string + var targetPort int + + g.By("Selecting a single EgressIP node, and one node per source deployment") + // Requires a minimum of 3 worker nodes in total: + // 1 nonEgressIPNodeName + at least 2 as sources of EgressIP traffic. + o.Expect(len(egressIPNodesOrderedNames)).Should(o.BeNumerically(">", 1)) + egressIPNodeStr := []string{egressIPNodesOrderedNames[0]} + deploymentNodeStr := [][]string{ + {egressIPNodesOrderedNames[0]}, + {egressIPNodesOrderedNames[1]}, + } + + g.By("Creating the target DaemonSet with a single hostnetworked pod on the target node") + daemonSetName := "hostnetworked" + // Try the entire port range to create the DaemonSet. + for i := 0; i < egressIPTargetHostPortMax-egressIPTargetHostPortMin; i++ { + containerPort, err := portAllocator.AllocateNextPort() + o.Expect(err).NotTo(o.HaveOccurred()) + + // use the port that we got from the port allocator for this + // new DS / pod. Store the created daemonset for later. + _, err = createHostNetworkedDaemonSetAndProbe( + clientset, + externalNamespace, + nonEgressIPNodeName, + daemonSetName, + containerPort, + 10, // every 10 seconds + 6, // for 6 retries + ) + + // If this is a port conflict, then keep the port allocation and + // simply continue (but delete the current DS first). + // The current port is hence marked as unavailable for + // further tries. + if err != nil && strings.Contains(err.Error(), "Port conflict when creating pod") { + err := deleteDaemonSet(clientset, externalNamespace, daemonSetName) + o.Expect(err).NotTo(o.HaveOccurred()) + continue + } + // Any other error shoud not have occurred. + o.Expect(err).NotTo(o.HaveOccurred()) + + // Break if no error was found. + targetPort = containerPort + break + } - // use the port that we got from the port allocator for this - // new DS / pod. Store the created daemonset for later. - _, err = createHostNetworkedDaemonSetAndProbe( - clientset, - externalNamespace, - nonEgressIPNodeName, - daemonSetName, - containerPort, - 10, // every 10 seconds - 6, // for 6 retries - ) - - // If this is a port conflict, then keep the port allocation and - // simply continue (but delete the current DS first). - // The current port is hence marked as unavailable for - // further tries. - if err != nil && strings.Contains(err.Error(), "Port conflict when creating pod") { - err := deleteDaemonSet(clientset, externalNamespace, daemonSetName) + g.By("Getting the targetIP for the test from the DaemonSet pod") + podIPs, err := getDaemonSetPodIPs(clientset, externalNamespace, daemonSetName) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(podIPs)).Should(o.BeNumerically(">", 0)) + targetIP = podIPs[0] + + var routeNames []string + for k, v := range deploymentNodeStr { + g.By(fmt.Sprintf("Creating EgressIP test source deployment %d with number of pods equals number of EgressIP nodes", k)) + _, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, fmt.Sprint(k), ingressDomain, len(v), v) + routeNames = append(routeNames, routeName) o.Expect(err).NotTo(o.HaveOccurred()) - continue } - // Any other error shoud not have occurred. + + // For this test, get a single EgressIP per node. + // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. + // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature + // will pick the actual node. + g.By("Getting a map of source nodes and potential Egress IPs for these nodes") + egressIPsPerNode := 1 + nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodeStr, cloudType, egressIPsPerNode) + framework.Logf("%v", nodeEgressIPMap) o.Expect(err).NotTo(o.HaveOccurred()) - // Break if no error was found. - targetPort = containerPort - break - } + g.By("Choosing the EgressIPs to be assigned, one per node") + egressIPSet := make(map[string]string) + for nodeName, eip := range nodeEgressIPMap { + _, ok := egressIPSet[eip[0]] + if !ok { + egressIPSet[eip[0]] = nodeName + } + } - g.By("Getting the targetIP for the test from the DaemonSet pod") - podIPs, err := getDaemonSetPodIPs(clientset, externalNamespace, daemonSetName) - o.Expect(err).NotTo(o.HaveOccurred()) - o.Expect(len(podIPs)).Should(o.BeNumerically(">", 0)) - targetIP = podIPs[0] - - var routeNames []string - for k, v := range deploymentNodeStr { - g.By(fmt.Sprintf("Creating EgressIP test source deployment %d with number of pods equals number of EgressIP nodes", k)) - _, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, fmt.Sprint(k), ingressDomain, len(v), v) - routeNames = append(routeNames, routeName) + g.By("Creating the EgressIP object") + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + egressIPObjectName := egressIPNamespace + createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) + + g.By("Applying the EgressIP object") + _, err = runOcWithRetry(oc.AsAdmin(), "create", "-f", tmpDirEgressIP+"/"+egressIPYaml) o.Expect(err).NotTo(o.HaveOccurred()) - } - // For this test, get a single EgressIP per node. - // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. - // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature - // will pick the actual node. - g.By("Getting a map of source nodes and potential Egress IPs for these nodes") - egressIPsPerNode := 1 - nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodeStr, cloudType, egressIPsPerNode) - framework.Logf("%v", nodeEgressIPMap) - o.Expect(err).NotTo(o.HaveOccurred()) + // This approach here is different from the other tests because: + // a) No additional SNAT or similar can be injected by the cloud as we go directly from node and we know that we have + // an endpoint on the cloud, always, thus we can directly query agnhost's /clientip. + // b) The requests in tcpdump did not expose the request string for some reason (probably needed better filters) + // c) It's simpler to just query for the /clientip instead of relying on the packet capture for these tests. + for _, routeName := range routeNames { + g.By(fmt.Sprintf("Launching a new prober pod and probing for EgressIPs at %s", routeName)) + numberOfRequestsToSend := 10 + clientIPSet, err := probeForClientIPs(oc, externalNamespace, probePodName, routeName, targetIP, targetPort, numberOfRequestsToSend) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Choosing the EgressIPs to be assigned, one per node") - egressIPSet := make(map[string]string) - for nodeName, eip := range nodeEgressIPMap { - _, ok := egressIPSet[eip[0]] - if !ok { - egressIPSet[eip[0]] = nodeName + // Note: my interpretation is that it's a bug if we see an egressIP here: + // We should never see egressIPs when querying internal targets: + // https://bugzilla.redhat.com/show_bug.cgi?id=2070929 + // However, this was still a subject of discussion. When we enable these tests after + // we fix 2070929, decide if we want to see EgressIPs here or not and possibly remove + // this verification. + g.By("Making sure that EgressIPs were not part of the response") + framework.Logf("egressIPSet is: %v", egressIPSet) + framework.Logf("clientIPSet is: %v", clientIPSet) + o.Expect(len(clientIPSet)).Should(o.BeNumerically(">", 0)) + o.Expect( + // return false if any key of x is in y or vice-versa. + func(x map[string]string, y map[string]struct{}) bool { + for k := range x { + if _, ok := y[k]; ok { + return false + } + } + for k := range y { + if _, ok := x[k]; ok { + return false + } + } + return true + }(egressIPSet, clientIPSet)).To(o.BeTrue()) } - } + }) + }) // end testing to internal targets + + g.Context("[external-targets][apigroup:user.openshift.io][apigroup:security.openshift.io]", func() { + g.JustBeforeEach(func() { + // SCC privileged is needed to run tcpdump on the packet sniffer containers, and at the minimum host networked is needed for + // host networked pods. + g.By("Adding SCC privileged to the external namespace") + _, err := runOcWithRetry(oc.AsAdmin(), "adm", "policy", "add-scc-to-user", "privileged", fmt.Sprintf("system:serviceaccount:%s:default", externalNamespace)) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Creating the EgressIP object") - egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml - egressIPObjectName := egressIPNamespace - createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) + g.By("Determining the interface that will be used for packet sniffing") + packetSnifferInterface, err = findPacketSnifferInterface(oc, egressIPNodesOrderedNames) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Using interface %s for packet captures", packetSnifferInterface) - g.By("Applying the EgressIP object") - _, err = runOcWithRetry(oc.AsAdmin(), "create", "-f", tmpDirEgressIP+"/"+egressIPYaml) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By("Spawning the packet sniffer pods on the EgressIP assignable hosts") + packetSnifferDaemonSet, err = createPacketSnifferDaemonSet(oc, externalNamespace, egressIPNodesOrderedNames, targetProtocol, targetPort, packetSnifferInterface) + o.Expect(err).NotTo(o.HaveOccurred()) + }) - // This approach here is different from the other tests because: - // a) No additional SNAT or similar can be injected by the cloud as we go directly from node and we know that we have - // an endpoint on the cloud, always, thus we can directly query agnhost's /clientip. - // b) The requests in tcpdump did not expose the request string for some reason (probably needed better filters) - // c) It's simpler to just query for the /clientip instead of relying on the packet capture for these tests. - for _, routeName := range routeNames { - g.By(fmt.Sprintf("Launching a new prober pod and probing for EgressIPs at %s", routeName)) - numberOfRequestsToSend := 10 - clientIPSet, err := probeForClientIPs(oc, externalNamespace, probePodName, routeName, targetIP, targetPort, numberOfRequestsToSend) + // Skipped on Azure due to https://bugzilla.redhat.com/show_bug.cgi?id=2073045 + g.It("pods should have the assigned EgressIPs and EgressIPs can be deleted and recreated [Skipped:azure][apigroup:route.openshift.io]", func() { + g.By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") + _, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) o.Expect(err).NotTo(o.HaveOccurred()) - // Note: my interpretation is that it's a bug if we see an egressIP here: - // We should never see egressIPs when querying internal targets: - // https://bugzilla.redhat.com/show_bug.cgi?id=2070929 - // However, this was still a subject of discussion. When we enable these tests after - // we fix 2070929, decide if we want to see EgressIPs here or not and possibly remove - // this verification. - g.By("Making sure that EgressIPs were not part of the response") - framework.Logf("egressIPSet is: %v", egressIPSet) - framework.Logf("clientIPSet is: %v", clientIPSet) - o.Expect(len(clientIPSet)).Should(o.BeNumerically(">", 0)) - o.Expect( - // return false if any key of x is in y or vice-versa. - func(x map[string]string, y map[string]struct{}) bool { - for k := range x { - if _, ok := y[k]; ok { - return false - } - } - for k := range y { - if _, ok := x[k]; ok { - return false - } - } - return true - }(egressIPSet, clientIPSet)).To(o.BeTrue()) - } - }) - }) // end testing to internal targets - - g.Context("[external-targets][apigroup:user.openshift.io][apigroup:security.openshift.io]", func() { - g.JustBeforeEach(func() { - // SCC privileged is needed to run tcpdump on the packet sniffer containers, and at the minimum host networked is needed for - // host networked pods. - g.By("Adding SCC privileged to the external namespace") - _, err := runOcWithRetry(oc.AsAdmin(), "adm", "policy", "add-scc-to-user", "privileged", fmt.Sprintf("system:serviceaccount:%s:default", externalNamespace)) - o.Expect(err).NotTo(o.HaveOccurred()) + // For this test, get a single EgressIP per node. + // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. + // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature + // will pick the actual node. + g.By("Getting a map of source nodes and potential Egress IPs for these nodes") + egressIPsPerNode := 1 + nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode) + framework.Logf("%v", nodeEgressIPMap) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Determining the interface that will be used for packet sniffing") - packetSnifferInterface, err = findPacketSnifferInterface(oc, egressIPNodesOrderedNames) - o.Expect(err).NotTo(o.HaveOccurred()) - framework.Logf("Using interface %s for packet captures", packetSnifferInterface) + g.By("Choosing the EgressIPs to be assigned, one per node") + egressIPSet := make(map[string]string) + for nodeName, eip := range nodeEgressIPMap { + _, ok := egressIPSet[eip[0]] + if !ok { + egressIPSet[eip[0]] = nodeName + } + } - g.By("Spawning the packet sniffer pods on the EgressIP assignable hosts") - packetSnifferDaemonSet, err = createPacketSnifferDaemonSet(oc, externalNamespace, egressIPNodesOrderedNames, targetProtocol, targetPort, packetSnifferInterface) - o.Expect(err).NotTo(o.HaveOccurred()) - }) + numberOfRequestsToSend := 10 + if targetHost == "self" { + targetHost = routeName + } + // Run this twice to make sure that repeated EgressIP creation and deletion works. + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + egressIPObjectName := egressIPNamespace + for i := 0; i < 2; i++ { + g.By("Creating the EgressIP object") + createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) - // Skipped on Azure due to https://bugzilla.redhat.com/show_bug.cgi?id=2073045 - g.It("pods should have the assigned EgressIPs and EgressIPs can be deleted and recreated [Skipped:azure][apigroup:route.openshift.io]", func() { - g.By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") - _, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By("Applying the EgressIP object") + applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) - // For this test, get a single EgressIP per node. - // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. - // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature - // will pick the actual node. - g.By("Getting a map of source nodes and potential Egress IPs for these nodes") - egressIPsPerNode := 1 - nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode) - framework.Logf("%v", nodeEgressIPMap) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + + g.By("Deleting the EgressIP object") + // Use cascading foreground deletion to make sure that the EgressIP object and its dependencies are gone. + _, err = runOcWithRetry(oc.AsAdmin(), "delete", "egressip", egressIPObjectName, "--cascade=foreground") + o.Expect(err).NotTo(o.HaveOccurred()) + + // Azure often fails on this step here - BZ https://bugzilla.redhat.com/show_bug.cgi?id=2073045 + g.By(fmt.Sprintf("Waiting for maximum %d seconds for the CloudPrivateIPConfig objects to vanish", egressUpdateTimeout)) + waitForCloudPrivateIPConfigsDeletion(oc, cloudNetworkClientset, egressIPSet, egressUpdateTimeout) - g.By("Choosing the EgressIPs to be assigned, one per node") - egressIPSet := make(map[string]string) - for nodeName, eip := range nodeEgressIPMap { - _, ok := egressIPSet[eip[0]] - if !ok { - egressIPSet[eip[0]] = nodeName + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", 0, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, 0, packetSnifferDaemonSet, egressIPSet) } - } - numberOfRequestsToSend := 10 - if targetHost == "self" { - targetHost = routeName - } - // Run this twice to make sure that repeated EgressIP creation and deletion works. - egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml - egressIPObjectName := egressIPNamespace - for i := 0; i < 2; i++ { + g.By("Removing the egressIPYaml file to signal that no further cleanup is needed") + os.Remove(egressIPYamlPath) + }) + + g.It("pods should keep the assigned EgressIPs when being rescheduled to another node", func() { + g.By("Selecting a single EgressIP node, and a single start node for the pod") + // requires a total of 3 worker nodes + o.Expect(len(egressIPNodesOrderedNames)).Should(o.BeNumerically(">", 1)) + leftNode := egressIPNodesOrderedNames[0:1] + rightNode := egressIPNodesOrderedNames[1:2] + + g.By(fmt.Sprintf("Creating the EgressIP test source deployment on node %s", rightNode[0])) + deploymentName, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "", ingressDomain, len(rightNode), rightNode) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Getting an EgressIP for a specific node only works on AWS. However, the important + // thing here is that we get only a single EgressIP which will be assigned to one + // of the 2 nodes only. On AWS, the EgressIP and the pod will end up on different nodes, + // the pod will then always be moved to the node that the EgressIP is on. On other cloud + // platforms, what happens depends on the involved controllers. Either, the pod and + // EgressIPs start out on the same node, or on different nodes. The end result though + // is that we always test both scenarios: pod and EgressIP on the same node, pod and + // EgressIP on different nodes. And we also test that pods can be moved between nodes. + g.By(fmt.Sprintf("Finding potential Egress IPs for node %s", leftNode[0])) + egressIPsPerNode := 1 + nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, leftNode, cloudType, egressIPsPerNode) + framework.Logf("%v", nodeEgressIPMap) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("Choosing the single EgressIP to be assigned") + egressIPSet := make(map[string]string) + for nodeName, eip := range nodeEgressIPMap { + _, ok := egressIPSet[eip[0]] + if !ok { + egressIPSet[eip[0]] = nodeName + } + } + + // This step is different depending on the network plugin. g.By("Creating the EgressIP object") + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + egressIPObjectName := egressIPNamespace createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) g.By("Applying the EgressIP object") applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) + numberOfRequestsToSend := 10 + if targetHost == "self" { + targetHost = routeName + } g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) - g.By("Deleting the EgressIP object") - // Use cascading foreground deletion to make sure that the EgressIP object and its dependencies are gone. - _, err = runOcWithRetry(oc.AsAdmin(), "delete", "egressip", egressIPObjectName, "--cascade=foreground") + g.By("Updating the source deployment's Affinity and moving it to the other source node") + err = updateDeploymentAffinity(oc, egressIPNamespace, deploymentName, leftNode) o.Expect(err).NotTo(o.HaveOccurred()) - // Azure often fails on this step here - BZ https://bugzilla.redhat.com/show_bug.cgi?id=2073045 - g.By(fmt.Sprintf("Waiting for maximum %d seconds for the CloudPrivateIPConfig objects to vanish", egressUpdateTimeout)) - waitForCloudPrivateIPConfigsDeletion(oc, cloudNetworkClientset, egressIPSet, egressUpdateTimeout) + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + }) - g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", 0, egressIPSet)) - spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, 0, packetSnifferDaemonSet, egressIPSet) - } + g.It("only pods matched by the pod selector should have the EgressIPs", func() { + g.By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") + deployment0Name, route0Name, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "0", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Removing the egressIPYaml file to signal that no further cleanup is needed") - os.Remove(egressIPYamlPath) - }) + g.By("Creating the second EgressIP test source deployment with number of pods equals number of EgressIP nodes") + _, route1Name, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "1", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) + o.Expect(err).NotTo(o.HaveOccurred()) - g.It("pods should keep the assigned EgressIPs when being rescheduled to another node", func() { - g.By("Selecting a single EgressIP node, and a single start node for the pod") - // requires a total of 3 worker nodes - o.Expect(len(egressIPNodesOrderedNames)).Should(o.BeNumerically(">", 1)) - leftNode := egressIPNodesOrderedNames[0:1] - rightNode := egressIPNodesOrderedNames[1:2] + // For this test, get a single EgressIP per node. + // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. + // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature + // will pick the actual node. + g.By("Getting a map of source nodes and potential Egress IPs for these nodes") + egressIPsPerNode := 1 + nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode) + framework.Logf("%v", nodeEgressIPMap) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By(fmt.Sprintf("Creating the EgressIP test source deployment on node %s", rightNode[0])) - deploymentName, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "", ingressDomain, len(rightNode), rightNode) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By("Choosing the EgressIPs to be assigned, one per node") + egressIPSet := make(map[string]string) + for nodeName, eip := range nodeEgressIPMap { + _, ok := egressIPSet[eip[0]] + if !ok { + egressIPSet[eip[0]] = nodeName + } + } - // Getting an EgressIP for a specific node only works on AWS. However, the important - // thing here is that we get only a single EgressIP which will be assigned to one - // of the 2 nodes only. On AWS, the EgressIP and the pod will end up on different nodes, - // the pod will then always be moved to the node that the EgressIP is on. On other cloud - // platforms, what happens depends on the involved controllers. Either, the pod and - // EgressIPs start out on the same node, or on different nodes. The end result though - // is that we always test both scenarios: pod and EgressIP on the same node, pod and - // EgressIP on different nodes. And we also test that pods can be moved between nodes. - g.By(fmt.Sprintf("Finding potential Egress IPs for node %s", leftNode[0])) - egressIPsPerNode := 1 - nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, leftNode, cloudType, egressIPsPerNode) - framework.Logf("%v", nodeEgressIPMap) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By("Creating the EgressIP object") + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + egressIPObjectName := egressIPNamespace + createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, fmt.Sprintf("app: %s", deployment0Name), egressIPSet) + + g.By("Applying the EgressIP object") + applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) - g.By("Choosing the single EgressIP to be assigned") - egressIPSet := make(map[string]string) - for nodeName, eip := range nodeEgressIPMap { - _, ok := egressIPSet[eip[0]] - if !ok { - egressIPSet[eip[0]] = nodeName + numberOfRequestsToSend := 10 + if targetHost == "self" { + targetHost = route0Name } - } + g.By(fmt.Sprintf("Testing first EgressIP test source deployment and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, route0Name, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) - // This step is different depending on the network plugin. - g.By("Creating the EgressIP object") - egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml - egressIPObjectName := egressIPNamespace - createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) + if targetHost == "self" { + targetHost = route1Name + } + g.By(fmt.Sprintf("Testing second EgressIP test source deployment and making sure that %d requests with search string and EgressIPs %v were seen", 0, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, route1Name, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, 0, packetSnifferDaemonSet, egressIPSet) + }) + + g.It("pods should have the assigned EgressIPs and EgressIPs can be updated", func() { + g.By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") + _, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Applying the EgressIP object") - applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) + // For this test, get a single EgressIP per node. + // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. + // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature + // will pick the actual node. + g.By("Getting a map of source nodes and potential Egress IPs for these nodes") + egressIPsPerNode := 1 + nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode) + framework.Logf("%v", nodeEgressIPMap) + o.Expect(err).NotTo(o.HaveOccurred()) - numberOfRequestsToSend := 10 - if targetHost == "self" { - targetHost = routeName - } - g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) - spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + g.By("Choosing the EgressIPs to be assigned, one per node, for a total of 2 nodes") + i := 0 + egressIPSetTemp := make(map[string]string) + for nodeName, eip := range nodeEgressIPMap { + // only do this for 2 nodes + if i > 1 { + break + } + i++ + + _, ok := egressIPSetTemp[eip[0]] + if !ok { + egressIPSetTemp[eip[0]] = nodeName + } + } + o.Expect(len(egressIPSetTemp)).Should(o.BeNumerically("==", 2)) - g.By("Updating the source deployment's Affinity and moving it to the other source node") - err = updateDeploymentAffinity(oc, egressIPNamespace, deploymentName, leftNode) - o.Expect(err).NotTo(o.HaveOccurred()) + // Run this for each of the EgressIPs (and because we are applying, this will update the EgressIP object) + numberOfRequestsToSend := 10 + if targetHost == "self" { + targetHost = routeName + } + for eip, nodeName := range egressIPSetTemp { + egressIPSet := map[string]string{eip: nodeName} - g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) - spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) - }) + g.By("Creating the EgressIP object") + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + egressIPObjectName := egressIPNamespace + createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) - g.It("only pods matched by the pod selector should have the EgressIPs", func() { - g.By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") - deployment0Name, route0Name, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "0", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By("Applying the EgressIP object") + applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) - g.By("Creating the second EgressIP test source deployment with number of pods equals number of EgressIP nodes") - _, route1Name, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "1", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) - o.Expect(err).NotTo(o.HaveOccurred()) + g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) + spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + } + }) + }) // end testing to external targets + }) // end cloud platform tests - // For this test, get a single EgressIP per node. - // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. - // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature - // will pick the actual node. - g.By("Getting a map of source nodes and potential Egress IPs for these nodes") - egressIPsPerNode := 1 - nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode) - framework.Logf("%v", nodeEgressIPMap) - o.Expect(err).NotTo(o.HaveOccurred()) + g.Context("EgressIP duplicate MAC prevention", func() { + const ( + egressIPObjectName = "egressip-mac-test" + ) - g.By("Choosing the EgressIPs to be assigned, one per node") - egressIPSet := make(map[string]string) - for nodeName, eip := range nodeEgressIPMap { - _, ok := egressIPSet[eip[0]] - if !ok { - egressIPSet[eip[0]] = nodeName + g.BeforeEach(func() { + g.By("Checking platform type - this test requires L2 network adjacency") + infra, err := oc.AdminConfigClient().ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + if infra.Status.PlatformStatus != nil { + platformType := infra.Status.PlatformStatus.Type + cloudPlatforms := []configv1.PlatformType{ + configv1.AWSPlatformType, + configv1.GCPPlatformType, + configv1.AzurePlatformType, + configv1.OpenStackPlatformType, + } + for _, cp := range cloudPlatforms { + if platformType == cp { + skipper.Skipf("This test requires L2 network adjacency (baremetal); cloud platform %s is not supported", platformType) + } } } + }) - g.By("Creating the EgressIP object") + g.AfterEach(func() { + g.By("Deleting the EgressIP object if it exists") egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml - egressIPObjectName := egressIPNamespace - createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, fmt.Sprintf("app: %s", deployment0Name), egressIPSet) - - g.By("Applying the EgressIP object") - applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) - - numberOfRequestsToSend := 10 - if targetHost == "self" { - targetHost = route0Name + if _, err := os.Stat(egressIPYamlPath); err == nil { + _, _ = runOcWithRetry(oc.AsAdmin(), "delete", "-f", egressIPYamlPath) } - g.By(fmt.Sprintf("Testing first EgressIP test source deployment and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) - spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, route0Name, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) - if targetHost == "self" { - targetHost = route1Name + g.By("Removing the egress-assignable labels from all worker nodes") + for _, nodeName := range workerNodesOrderedNames { + _, _ = runOcWithRetry(oc.AsAdmin(), "label", "node", nodeName, "k8s.ovn.org/egress-assignable-") } - g.By(fmt.Sprintf("Testing second EgressIP test source deployment and making sure that %d requests with search string and EgressIPs %v were seen", 0, egressIPSet)) - spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, route1Name, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, 0, packetSnifferDaemonSet, egressIPSet) }) - g.It("pods should have the assigned EgressIPs and EgressIPs can be updated", func() { - g.By("Creating the EgressIP test source deployment with number of pods equals number of EgressIP nodes") - _, routeName, err := createAgnhostDeploymentAndIngressRoute(oc, egressIPNamespace, "", ingressDomain, len(egressIPNodesOrderedNames), egressIPNodesOrderedNames) + g.It("should prevent duplicate MAC responses when egress node is rebooted [Serial]", func() { + // Node assignment: + // workerNodesOrderedNames[0] = probe node (runs arping, NOT egress-assignable) + // workerNodesOrderedNames[1] = egress node 1 (initial EgressIP holder) + // workerNodesOrderedNames[2] = egress node 2 (failover target) + probeNodeName := workerNodesOrderedNames[0] + egressNode1Name := workerNodesOrderedNames[1] + egressNode2Name := workerNodesOrderedNames[2] + + g.By("1. Labeling egress node 1 as egress-assignable") + _, err := runOcWithRetry(oc.AsAdmin(), "label", "node", egressNode1Name, "k8s.ovn.org/egress-assignable=") o.Expect(err).NotTo(o.HaveOccurred()) - // For this test, get a single EgressIP per node. - // Note: On some clouds like GCP, there is no dedicated CIDR per node and instead all EgressIPs come from a common pool. - // Thus, this is only an artificial assignment of EgressIP to node on these cloud platforms and the EgressIP feature - // will pick the actual node. - g.By("Getting a map of source nodes and potential Egress IPs for these nodes") - egressIPsPerNode := 1 - nodeEgressIPMap, err := findNodeEgressIPs(oc, clientset, cloudNetworkClientset, egressIPNodesOrderedNames, cloudType, egressIPsPerNode) - framework.Logf("%v", nodeEgressIPMap) + g.By("2. Allocating an EgressIP from egress node 1") + nodeEgressIPMap, err := findNodeEgressIPsBaremetal(oc, clientset, []string{egressNode1Name}) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(nodeEgressIPMap).To(o.HaveKey(egressNode1Name)) + o.Expect(nodeEgressIPMap[egressNode1Name]).NotTo(o.BeEmpty(), + fmt.Sprintf("no free EgressIP found for node %s", egressNode1Name)) + egressIPStr := nodeEgressIPMap[egressNode1Name][0] + egressIPIsIPv6 := net.ParseIP(egressIPStr).To4() == nil + framework.Logf("Allocated EgressIP: %s (IPv6: %v) for node %s", egressIPStr, egressIPIsIPv6, egressNode1Name) + + g.By("3. Creating and applying the EgressIP object") + egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml + egressIPSet := map[string]string{egressIPStr: egressNode1Name} + createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, oc.Namespace(), "", egressIPSet) + _, err = runOcWithRetry(oc.AsAdmin(), "create", "-f", egressIPYamlPath) o.Expect(err).NotTo(o.HaveOccurred()) - g.By("Choosing the EgressIPs to be assigned, one per node, for a total of 2 nodes") - i := 0 - egressIPSetTemp := make(map[string]string) - for nodeName, eip := range nodeEgressIPMap { - // only do this for 2 nodes - if i > 1 { - break + g.By("4. Verifying EgressIP is assigned to egress node 1") + var hasIP bool + var assignedNode string + o.Eventually(func() bool { + hasIP, assignedNode, err = egressIPStatusHasIP(oc, egressIPObjectName, egressIPStr) + if err != nil { + framework.Logf("Error checking EgressIP status: %v", err) + return false } - i++ + return hasIP && assignedNode == egressNode1Name + }, 60*time.Second, 5*time.Second).Should(o.BeTrue(), + fmt.Sprintf("EgressIP %s should be assigned to node %s", egressIPStr, egressNode1Name)) + framework.Logf("EgressIP %s assigned to node: %s", egressIPStr, assignedNode) - _, ok := egressIPSetTemp[eip[0]] - if !ok { - egressIPSetTemp[eip[0]] = nodeName - } - } - o.Expect(len(egressIPSetTemp)).Should(o.BeNumerically("==", 2)) + g.By("5. Labeling egress node 2 as egress-assignable for failover") + _, err = runOcWithRetry(oc.AsAdmin(), "label", "node", egressNode2Name, "k8s.ovn.org/egress-assignable=") + o.Expect(err).NotTo(o.HaveOccurred()) - // Run this for each of the EgressIPs (and because we are applying, this will update the EgressIP object) - numberOfRequestsToSend := 10 - if targetHost == "self" { - targetHost = routeName - } - for eip, nodeName := range egressIPSetTemp { - egressIPSet := map[string]string{eip: nodeName} + g.By("6. Getting br-ex physical interface name on each node") + iface1, err := findBridgePhysicalInterface(oc, egressNode1Name, "br-ex") + o.Expect(err).NotTo(o.HaveOccurred()) + iface2, err := findBridgePhysicalInterface(oc, egressNode2Name, "br-ex") + o.Expect(err).NotTo(o.HaveOccurred()) + probeInterface, err := findBridgePhysicalInterface(oc, probeNodeName, "br-ex") + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Physical interfaces - node1: %s, node2: %s, probe: %s", iface1, iface2, probeInterface) - g.By("Creating the EgressIP object") - egressIPYamlPath := tmpDirEgressIP + "/" + egressIPYaml - egressIPObjectName := egressIPNamespace - createEgressIPObject(oc, egressIPYamlPath, egressIPObjectName, egressIPNamespace, "", egressIPSet) + g.By("7. Getting MAC addresses of egress node 1 and egress node 2") + mac1, err := getNodeInterfaceMAC(oc, egressNode1Name, iface1) + o.Expect(err).NotTo(o.HaveOccurred()) + mac2, err := getNodeInterfaceMAC(oc, egressNode2Name, iface2) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Egress node 1 (%s) MAC: %s", egressNode1Name, mac1) + framework.Logf("Egress node 2 (%s) MAC: %s", egressNode2Name, mac2) - g.By("Applying the EgressIP object") - applyEgressIPObject(oc, cloudNetworkClientset, egressIPYamlPath, egressIPNamespace, egressIPSet, egressUpdateTimeout) + g.By("8. Verifying EgressIP resolves to egress node 1 MAC before migration") + probePodInfo, err := ovnkubePod(oc, probeNodeName) + o.Expect(err).NotTo(o.HaveOccurred()) - g.By(fmt.Sprintf("Sending requests from prober and making sure that %d requests with search string and EgressIPs %v were seen", numberOfRequestsToSend, egressIPSet)) - spawnProberSendEgressIPTrafficCheckLogs(oc, externalNamespace, probePodName, routeName, targetProtocol, targetHost, targetPort, numberOfRequestsToSend, numberOfRequestsToSend, packetSnifferDaemonSet, egressIPSet) + var discoveryCmd string + var macRegex *regexp.Regexp + if egressIPIsIPv6 { + discoveryCmd = fmt.Sprintf("ndisc6 -1 -w 1000 %s %s 2>&1", egressIPStr, probeInterface) + macRegex = regexp.MustCompile(`Target link-layer address:\s+([0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2})`) + } else { + discoveryCmd = fmt.Sprintf("arping -c 1 -I %s %s 2>&1", probeInterface, egressIPStr) + macRegex = regexp.MustCompile(`\[([0-9a-fA-F:]+)\]`) + } + output, err := adminExecInPod(oc, "openshift-ovn-kubernetes", probePodInfo.podName, probePodInfo.containerName, discoveryCmd) + o.Expect(err).NotTo(o.HaveOccurred(), "network discovery should succeed before migration") + matches := macRegex.FindStringSubmatch(output) + o.Expect(matches).To(o.HaveLen(2), fmt.Sprintf("should extract MAC from discovery output: %s", output)) + macBeforeMigration := strings.ToLower(strings.TrimSpace(matches[1])) + framework.Logf("MAC before migration: %s, expected node 1 MAC: %s", macBeforeMigration, mac1) + o.Expect(macBeforeMigration).To(o.Equal(mac1), "EgressIP should resolve to egress node 1 MAC before migration") + + g.By("9. Getting ovnkube-node pod name on egress node 1") + egressNode1PodInfo, err := ovnkubePod(oc, egressNode1Name) + o.Expect(err).NotTo(o.HaveOccurred()) + framework.Logf("Found ovnkube-node pod: %s on node %s", egressNode1PodInfo.podName, egressNode1Name) + + g.By("10. Starting goroutine to monitor nftables chain creation during pod deletion") + nftChainFound := make(chan bool, 1) + stopChecking := make(chan bool, 1) + goroutineReady := make(chan bool, 1) + nftChainCheckCmd := "nft list chains 2>/dev/null | grep -q egressip-drop && echo FOUND || echo NOTFOUND" + go func() { + defer close(nftChainFound) + goroutineReady <- true + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + select { + case <-stopChecking: + return + case <-ticker.C: + // Use oc debug to run on the node directly since the ovnkube-node pod may be terminating + result, debugErr := oc.AsAdmin().Run("debug").Args( + "node/"+egressNode1Name, + "--", + "chroot", "/host", + "/bin/bash", "-c", + nftChainCheckCmd, + ).Output() + if debugErr == nil && strings.Contains(result, "FOUND") { + nftChainFound <- true + return + } + } + } + }() + <-goroutineReady + defer close(stopChecking) + framework.Logf("Nftables chain monitoring goroutine started") + + g.By("11. Deleting ovnkube-node pod on egress node 1 to trigger nftables rules and EgressIP migration") + framework.Logf("Deleting ovnkube-node pod %s to trigger EgressIP migration", egressNode1PodInfo.podName) + err = clientset.CoreV1().Pods("openshift-ovn-kubernetes").Delete(context.TODO(), egressNode1PodInfo.podName, metav1.DeleteOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("12. Verifying nftables chain egressip-drop exists on egress node 1 during shutdown") + select { + case found := <-nftChainFound: + o.Expect(found).To(o.BeTrue(), "nftables chain egressip-drop should be found on egress node 1") + framework.Logf("Nftables chain egressip-drop verified on node %s", egressNode1Name) + case <-time.After(60 * time.Second): + framework.Failf("Timed out waiting for nftables chain egressip-drop on node %s", egressNode1Name) } + + g.By("13. Waiting for EgressIP to migrate to egress node 2") + o.Eventually(func() bool { + hasIP, assignedNode, err = egressIPStatusHasIP(oc, egressIPObjectName, egressIPStr) + if err != nil { + framework.Logf("Error checking EgressIP status: %v", err) + return false + } + if hasIP && assignedNode == egressNode2Name { + return true + } + framework.Logf("EgressIP %s still on node %s, waiting for migration to %s", egressIPStr, assignedNode, egressNode2Name) + return false + }, 120*time.Second, 5*time.Second).Should(o.BeTrue(), + fmt.Sprintf("EgressIP %s should migrate to node %s", egressIPStr, egressNode2Name)) + framework.Logf("EgressIP successfully migrated to node %s", egressNode2Name) + + g.By("14. Checking for duplicate MAC responses after migration") + err = checkForDuplicateMACOnNode( + oc, + probeNodeName, + probeInterface, + egressIPStr, + mac1, + mac2, + egressIPIsIPv6, + 20, + 500*time.Millisecond, + ) + o.Expect(err).NotTo(o.HaveOccurred(), "duplicate MAC detection check failed") + + g.By("15. Waiting for ovnkube-node pod to restart on egress node 1") + o.Eventually(func() bool { + pods, listErr := clientset.CoreV1().Pods("openshift-ovn-kubernetes").List(context.TODO(), metav1.ListOptions{ + FieldSelector: fmt.Sprintf("spec.nodeName=%s", egressNode1Name), + LabelSelector: "app=ovnkube-node", + }) + if listErr != nil { + return false + } + for _, p := range pods.Items { + if p.Status.Phase == corev1.PodRunning && p.DeletionTimestamp == nil { + for _, c := range p.Status.ContainerStatuses { + if c.Ready { + return true + } + } + } + } + return false + }, 120*time.Second, 5*time.Second).Should(o.BeTrue(), + "ovnkube-node pod should restart on egress node 1") + framework.Logf("ovnkube-node pod restarted on node %s", egressNode1Name) + + g.By("16. Verifying nftables cleanup on egress node 1 after pod restart") + newPodInfo, err := ovnkubePod(oc, egressNode1Name) + o.Expect(err).NotTo(o.HaveOccurred()) + nftCheckCmd := "nft list table netdev ovn-kubernetes-egressip 2>&1 || true" + nftOutput, nftErr := adminExecInPod(oc, "openshift-ovn-kubernetes", newPodInfo.podName, newPodInfo.containerName, nftCheckCmd) + o.Expect(nftErr).NotTo(o.HaveOccurred()) + o.Expect(nftOutput).To(o.ContainSubstring("No such file"), + "nftables egress IP table should be deleted after cleanup") + framework.Logf("Nftables table cleaned up on node %s", egressNode1Name) + + framework.Logf("Test passed: EgressIP migrated cleanly without duplicate MAC responses") }) - }) // end testing to external targets + }) // end EgressIP duplicate MAC prevention }) // diff --git a/test/extended/networking/egressip_helpers.go b/test/extended/networking/egressip_helpers.go index b0b76a7531ce..b82624186b0c 100644 --- a/test/extended/networking/egressip_helpers.go +++ b/test/extended/networking/egressip_helpers.go @@ -1738,3 +1738,166 @@ func getEgressIP(oc *exutil.CLI, name string) (*EgressIP, error) { } return egressip, nil } + +// getNodeInterfaceMAC returns the MAC address of a network interface on a node +// by exec'ing `ip link show ` in the ovnkube-node pod. +func getNodeInterfaceMAC(oc *exutil.CLI, nodeName, interfaceName string) (string, error) { + ovnkubePodInfo, err := ovnkubePod(oc, nodeName) + if err != nil { + return "", fmt.Errorf("failed to find ovnkube-node pod on node %s: %v", nodeName, err) + } + + cmd := fmt.Sprintf("ip link show %s", interfaceName) + output, err := adminExecInPod(oc, "openshift-ovn-kubernetes", ovnkubePodInfo.podName, ovnkubePodInfo.containerName, cmd) + if err != nil { + return "", fmt.Errorf("failed to get interface %s info on node %s: %v", interfaceName, nodeName, err) + } + + macRegex := regexp.MustCompile(`link/ether\s+([0-9a-fA-F:]+)`) + matches := macRegex.FindStringSubmatch(output) + if len(matches) < 2 { + return "", fmt.Errorf("could not parse MAC from ip link show output on node %s: %s", nodeName, output) + } + return strings.ToLower(strings.TrimSpace(matches[1])), nil +} + +// checkForDuplicateMACOnNode performs arping (IPv4) or ndisc6 (IPv6) checks from a probe node's +// ovnkube-node pod to detect duplicate MAC address responses after EgressIP migration. +// Returns error immediately if old node MAC is detected responding. +func checkForDuplicateMACOnNode(oc *exutil.CLI, probeNodeName, interfaceName, egressIP, oldMAC, expectedMAC string, isIPv6 bool, maxChecks int, checkInterval time.Duration) error { + macRegexArping := regexp.MustCompile(`\[([0-9a-fA-F:]+)\]`) + macRegexNdisc6 := regexp.MustCompile(`Target link-layer address:\s+([0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2}:[0-9a-fA-F]{1,2})`) + + oldMAC = strings.ToLower(oldMAC) + expectedMAC = strings.ToLower(expectedMAC) + + probePodInfo, err := ovnkubePod(oc, probeNodeName) + if err != nil { + return fmt.Errorf("failed to find ovnkube-node pod on probe node %s: %v", probeNodeName, err) + } + + var toolName string + if isIPv6 { + toolName = "ndisc6" + } else { + toolName = "arping" + } + + whichCmd := fmt.Sprintf("which %s 2>&1", toolName) + _, whichErr := adminExecInPod(oc, "openshift-ovn-kubernetes", probePodInfo.podName, probePodInfo.containerName, whichCmd) + if whichErr != nil { + return fmt.Errorf("required binary %s not found in pod %s on node %s", toolName, probePodInfo.podName, probeNodeName) + } + + framework.Logf("Checking for duplicate MAC responses using %s from node %s (old MAC: %s, expected MAC: %s)...", + toolName, probeNodeName, oldMAC, expectedMAC) + + foundExpected := false + for i := 0; i < maxChecks; i++ { + var cmd string + var macRegex *regexp.Regexp + + if isIPv6 { + cmd = fmt.Sprintf("ndisc6 -1 -w 1000 %s %s 2>&1", egressIP, interfaceName) + macRegex = macRegexNdisc6 + } else { + cmd = fmt.Sprintf("arping -c 1 -I %s %s 2>&1", interfaceName, egressIP) + macRegex = macRegexArping + } + + output, execErr := adminExecInPod(oc, "openshift-ovn-kubernetes", probePodInfo.podName, probePodInfo.containerName, cmd) + if execErr != nil { + framework.Logf("Check %d/%d: %s command returned error: %v; output: %s", i+1, maxChecks, toolName, execErr, output) + } + + matches := macRegex.FindStringSubmatch(output) + if len(matches) >= 2 { + respondingMAC := strings.ToLower(strings.TrimSpace(matches[1])) + if respondingMAC == oldMAC { + return fmt.Errorf("DUPLICATE MAC DETECTED on check %d: Old node MAC %s responded to %s for egress IP %s after migration. "+ + "The nftables drop rules should have prevented this response", i+1, oldMAC, toolName, egressIP) + } else if respondingMAC == expectedMAC { + foundExpected = true + framework.Logf("Check %d/%d: New node MAC %s is responding (expected)", i+1, maxChecks, expectedMAC) + } else { + return fmt.Errorf("unexpected MAC %s (not old %s or expected %s)", respondingMAC, oldMAC, expectedMAC) + } + } + + if i < maxChecks-1 { + time.Sleep(checkInterval) + } + } + + if !foundExpected { + return fmt.Errorf("did not observe expected MAC %s responding to %s for egress IP %s after %d checks", + expectedMAC, toolName, egressIP, maxChecks) + } + framework.Logf("No duplicate MAC detected - nftables rules successfully blocked responses from old node") + return nil +} + +// findNodeEgressIPsBaremetal allocates EgressIPs from node egress-ipconfig annotations +// without requiring the CloudPrivateIPConfig CRD (which only exists on cloud platforms). +func findNodeEgressIPsBaremetal(oc *exutil.CLI, clientset kubernetes.Interface, nodeNames []string) (map[string][]string, error) { + var reservedIPs []string + + egressipList, err := listEgressIPs(oc) + if err != nil { + return nil, fmt.Errorf("failed to list EgressIPs: %v", err) + } + for _, egressip := range egressipList.Items { + reservedIPs = append(reservedIPs, egressip.Spec.EgressIPs...) + } + + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + for _, node := range nodes.Items { + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + reservedIPs = append(reservedIPs, addr.Address) + } + } + } + + nodeEgressIPs := make(map[string][]string) + for _, nodeName := range nodeNames { + node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + nodeEgressIPConfigs, err := getNodeEgressIPConfiguration(node) + if err != nil { + return nil, fmt.Errorf("failed to get egress IP configuration for node %s: %v", nodeName, err) + } + if l := len(nodeEgressIPConfigs); l != 1 { + return nil, fmt.Errorf("unexpected length of egress IP configuration for node %s: %d", nodeName, l) + } + ipnetStr := nodeEgressIPConfigs[0].IFAddr.IPv4 + if ipnetStr == "" { + ipnetStr = nodeEgressIPConfigs[0].IFAddr.IPv6 + if ipnetStr != "" { + _, ipnet, parseErr := net.ParseCIDR(ipnetStr) + if parseErr != nil { + return nil, fmt.Errorf("failed to parse IPv6 CIDR %s for node %s: %v", ipnetStr, nodeName, parseErr) + } + ones, _ := ipnet.Mask.Size() + if ones < 120 { + return nil, fmt.Errorf("IPv6 egress CIDR %s on node %s has prefix /%d which is too large to enumerate; minimum /120 required", ipnetStr, nodeName, ones) + } + } + } + freeIPs, err := getFirstFreeIPs(ipnetStr, reservedIPs, configv1.NonePlatformType, 1) + if err != nil { + return nil, fmt.Errorf("failed to find free EgressIP for node %s: %v", nodeName, err) + } + nodeEgressIPs[nodeName] = freeIPs + for _, freeIP := range freeIPs { + reservedIPs = append(reservedIPs, freeIP) + } + } + + return nodeEgressIPs, nil +}