From 9e8b6b9d1c0661bc64255245f218278f85c57d62 Mon Sep 17 00:00:00 2001 From: Miguel Angel Ajo Pelayo Date: Mon, 3 Nov 2025 21:49:50 +0000 Subject: [PATCH] operator: e2e ingress --- deploy/operator/test/e2e/e2e_test.go | 93 ++++++++++++++++++++++++++++ hack/deploy_with_operator.sh | 4 +- hack/kind_cluster.yaml | 9 +-- hack/utils | 3 +- 4 files changed, 101 insertions(+), 8 deletions(-) diff --git a/deploy/operator/test/e2e/e2e_test.go b/deploy/operator/test/e2e/e2e_test.go index 7a5206a3..cce798b1 100644 --- a/deploy/operator/test/e2e/e2e_test.go +++ b/deploy/operator/test/e2e/e2e_test.go @@ -29,6 +29,7 @@ import ( appsv1 "k8s.io/api/apps/v1" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -588,6 +589,98 @@ provisioning: Eventually(routerServicesCount, 1*time.Minute).Should(Equal(1)) }) + It("should setup ingress for the controller and router for ingress mode", func() { + By("updating the Jumpstarter custom resource to enable ingress mode") + jumpstarter := &operatorv1alpha1.Jumpstarter{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter", + Namespace: dynamicTestNamespace, + }, jumpstarter) + Expect(err).NotTo(HaveOccurred()) + + jumpstarter.Spec.Controller.GRPC.Endpoints = []operatorv1alpha1.Endpoint{ + { + Address: "grpc.jumpstarter.127.0.0.1.nip.io:5443", + Ingress: &operatorv1alpha1.IngressConfig{ + Enabled: true, + Class: "nginx", + }, + }, + } + jumpstarter.Spec.Routers.GRPC.Endpoints = []operatorv1alpha1.Endpoint{ + { + Address: "router.jumpstarter.127.0.0.1.nip.io:5443", + Ingress: &operatorv1alpha1.IngressConfig{ + Enabled: true, + Class: "nginx", + }, + }, + } + err = k8sClient.Update(ctx, jumpstarter) + Expect(err).NotTo(HaveOccurred()) + + By("verifying the ingress for the controller was created") + verifyIngressForController := func(g Gomega) bool { + ingress := &networkingv1.Ingress{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "controller-grpc-ing", + Namespace: dynamicTestNamespace, + }, ingress) + if err != nil { + return false + } + Expect(ingress.Spec.Rules).To(HaveLen(1)) + Expect(ingress.Spec.Rules[0].Host).To(Equal("grpc.jumpstarter.127.0.0.1.nip.io")) + Expect(ingress.Spec.Rules[0].HTTP.Paths).To(HaveLen(1)) + Expect(ingress.Spec.Rules[0].HTTP.Paths[0].Path).To(Equal("/")) + Expect(*ingress.Spec.Rules[0].HTTP.Paths[0].PathType).To(Equal(networkingv1.PathTypePrefix)) + return true + } + Eventually(verifyIngressForController, 1*time.Minute).Should(BeTrue()) + + By("verifying the ingress for the router was created") + verifyIngressForRouter := func(g Gomega) bool { + ingress := &networkingv1.Ingress{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter-router-0-ing", + Namespace: dynamicTestNamespace, + }, ingress) + if err != nil { + return false + } + Expect(ingress.Spec.Rules).To(HaveLen(1)) + Expect(ingress.Spec.Rules[0].Host).To(Equal("router.jumpstarter.127.0.0.1.nip.io")) + Expect(ingress.Spec.Rules[0].HTTP.Paths).To(HaveLen(1)) + Expect(ingress.Spec.Rules[0].HTTP.Paths[0].Path).To(Equal("/")) + Expect(*ingress.Spec.Rules[0].HTTP.Paths[0].PathType).To(Equal(networkingv1.PathTypePrefix)) + return true + } + Eventually(verifyIngressForRouter, 1*time.Minute).Should(BeTrue()) + }) + + It("should contain the right router configuration in the configmap", func() { + By("checking the configmap contains the right router configuration") + Eventually(func(g Gomega) string { + configmap := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Name: "jumpstarter-controller", + Namespace: dynamicTestNamespace, + }, configmap) + g.Expect(err).NotTo(HaveOccurred()) + return configmap.Data["router"] + }, 1*time.Minute).Should(ContainSubstring("router.jumpstarter.127.0.0.1.nip.io:5443")) + }) + + It("should allow access to ingress grpc endpoints", func() { + // TODO: fix ingress in kind (not working for helm either) + Skip("nginx ingress not working in kind") + + By("checking endpoint grpc access to controller") + waitForGRPCEndpoint("grpc.jumpstarter.127.0.0.1.nip.io:5443", 1*time.Minute) + By("checking endpoint grpc access to router") + waitForGRPCEndpoint("router.jumpstarter.127.0.0.1.nip.io:5443", 1*time.Minute) + }) + AfterAll(func() { DeleteTestNamespace(dynamicTestNamespace) }) diff --git a/hack/deploy_with_operator.sh b/hack/deploy_with_operator.sh index ae9374f0..84e5a14e 100755 --- a/hack/deploy_with_operator.sh +++ b/hack/deploy_with_operator.sh @@ -59,14 +59,14 @@ kubectl create namespace jumpstarter-lab --dry-run=client -o yaml | kubectl appl # Generate endpoint configuration based on networking mode if [ "${NETWORKING_MODE}" == "ingress" ]; then CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END - - address: grpc.${BASEDOMAIN}:443 + - address: grpc.${BASEDOMAIN}:5443 ingress: enabled: true class: "nginx" END ) ROUTER_ENDPOINT_CONFIG=$(cat <<-END - - address: router.${BASEDOMAIN}:443 + - address: router.${BASEDOMAIN}:5443 ingress: enabled: true class: "nginx" diff --git a/hack/kind_cluster.yaml b/hack/kind_cluster.yaml index 7ffd60b1..6478b336 100644 --- a/hack/kind_cluster.yaml +++ b/hack/kind_cluster.yaml @@ -17,6 +17,9 @@ nodes: - containerPort: 80 # ingress controller hostPort: 5080 protocol: TCP + - containerPort: 443 + hostPort: 5443 + protocol: TCP - containerPort: 30010 # grpc nodeport hostPort: 8082 protocol: TCP @@ -29,11 +32,9 @@ nodes: - containerPort: 30013 # grpc router nodeport (replica 2) hostPort: 8085 protocol: TCP - - containerPort: 443 - hostPort: 5443 - protocol: TCP + # if we needed to mount a hostPath volume into the kind cluster, we can do it like this # extraMounts: # - hostPath: ./bin/e2e-certs # containerPath: /tmp/e2e-certs - + diff --git a/hack/utils b/hack/utils index 88d77012..03ad9564 100755 --- a/hack/utils +++ b/hack/utils @@ -68,8 +68,7 @@ install_nginx_ingress() { lsmod | grep ip_tables || \ (echo "ip_tables module not loaded needed by nginx ingress, please run 'sudo modprobe ip_tables'" && exit 1) - # Deploy nginx ingress for kind - kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml +kubectl apply -f https://github.com/kubernetes/ingress-nginx/raw/refs/heads/main/deploy/static/provider/kind/deploy.yaml echo -e "${GREEN}Waiting for nginx to be ready ...${NC}"