operator: e2e ingress

deploy/operator/test/e2e/e2e_test.go

@@ -29,6 +29,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	authenticationv1 "k8s.io/api/authentication/v1"
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -588,6 +589,98 @@ provisioning:
 			Eventually(routerServicesCount, 1*time.Minute).Should(Equal(1))
 		})
 
+		It("should setup ingress for the controller and router for ingress mode", func() {
+			By("updating the Jumpstarter custom resource to enable ingress mode")
+			jumpstarter := &operatorv1alpha1.Jumpstarter{}
+			err := k8sClient.Get(ctx, types.NamespacedName{
+				Name:      "jumpstarter",
+				Namespace: dynamicTestNamespace,
+			}, jumpstarter)
+			Expect(err).NotTo(HaveOccurred())
+
+			jumpstarter.Spec.Controller.GRPC.Endpoints = []operatorv1alpha1.Endpoint{
+				{
+					Address: "grpc.jumpstarter.127.0.0.1.nip.io:5443",
+					Ingress: &operatorv1alpha1.IngressConfig{
+						Enabled: true,
+						Class:   "nginx",
+					},
+				},
+			}
+			jumpstarter.Spec.Routers.GRPC.Endpoints = []operatorv1alpha1.Endpoint{
+				{
+					Address: "router.jumpstarter.127.0.0.1.nip.io:5443",
+					Ingress: &operatorv1alpha1.IngressConfig{
+						Enabled: true,
+						Class:   "nginx",
+					},
+				},
+			}
+			err = k8sClient.Update(ctx, jumpstarter)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("verifying the ingress for the controller was created")
+			verifyIngressForController := func(g Gomega) bool {
+				ingress := &networkingv1.Ingress{}
+				err := k8sClient.Get(ctx, types.NamespacedName{
+					Name:      "controller-grpc-ing",
+					Namespace: dynamicTestNamespace,
+				}, ingress)
+				if err != nil {
+					return false
+				}
+				Expect(ingress.Spec.Rules).To(HaveLen(1))
+				Expect(ingress.Spec.Rules[0].Host).To(Equal("grpc.jumpstarter.127.0.0.1.nip.io"))
+				Expect(ingress.Spec.Rules[0].HTTP.Paths).To(HaveLen(1))
+				Expect(ingress.Spec.Rules[0].HTTP.Paths[0].Path).To(Equal("/"))
+				Expect(*ingress.Spec.Rules[0].HTTP.Paths[0].PathType).To(Equal(networkingv1.PathTypePrefix))
+				return true
+			}
+			Eventually(verifyIngressForController, 1*time.Minute).Should(BeTrue())
+
+			By("verifying the ingress for the router was created")
+			verifyIngressForRouter := func(g Gomega) bool {
+				ingress := &networkingv1.Ingress{}
+				err := k8sClient.Get(ctx, types.NamespacedName{
+					Name:      "jumpstarter-router-0-ing",
+					Namespace: dynamicTestNamespace,
+				}, ingress)
+				if err != nil {
+					return false
+				}
+				Expect(ingress.Spec.Rules).To(HaveLen(1))
+				Expect(ingress.Spec.Rules[0].Host).To(Equal("router.jumpstarter.127.0.0.1.nip.io"))
+				Expect(ingress.Spec.Rules[0].HTTP.Paths).To(HaveLen(1))
+				Expect(ingress.Spec.Rules[0].HTTP.Paths[0].Path).To(Equal("/"))
+				Expect(*ingress.Spec.Rules[0].HTTP.Paths[0].PathType).To(Equal(networkingv1.PathTypePrefix))
+				return true
+			}
+			Eventually(verifyIngressForRouter, 1*time.Minute).Should(BeTrue())
+		})
+
+		It("should contain the right router configuration in the configmap", func() {
+			By("checking the configmap contains the right router configuration")
+			Eventually(func(g Gomega) string {
+				configmap := &corev1.ConfigMap{}
+				err := k8sClient.Get(ctx, types.NamespacedName{
+					Name:      "jumpstarter-controller",
+					Namespace: dynamicTestNamespace,
+				}, configmap)
+				g.Expect(err).NotTo(HaveOccurred())
+				return configmap.Data["router"]
+			}, 1*time.Minute).Should(ContainSubstring("router.jumpstarter.127.0.0.1.nip.io:5443"))
+		})
+
+		It("should allow access to ingress grpc endpoints", func() {
+			// TODO: fix ingress in kind (not working for helm either)
+			Skip("nginx ingress not working in kind")
+
+			By("checking endpoint grpc access to controller")
+			waitForGRPCEndpoint("grpc.jumpstarter.127.0.0.1.nip.io:5443", 1*time.Minute)
+			By("checking endpoint grpc access to router")
+			waitForGRPCEndpoint("router.jumpstarter.127.0.0.1.nip.io:5443", 1*time.Minute)
+		})
+
 		AfterAll(func() {
 			DeleteTestNamespace(dynamicTestNamespace)
 		})

hack/deploy_with_operator.sh

@@ -59,14 +59,14 @@ kubectl create namespace jumpstarter-lab --dry-run=client -o yaml | kubectl appl
 # Generate endpoint configuration based on networking mode
 if [ "${NETWORKING_MODE}" == "ingress" ]; then
   CONTROLLER_ENDPOINT_CONFIG=$(cat <<-END
-        - address: grpc.${BASEDOMAIN}:443
+        - address: grpc.${BASEDOMAIN}:5443
           ingress:
             enabled: true
             class: "nginx"
 END
 )
   ROUTER_ENDPOINT_CONFIG=$(cat <<-END
-        - address: router.${BASEDOMAIN}:443
+        - address: router.${BASEDOMAIN}:5443
           ingress:
             enabled: true
             class: "nginx"

hack/kind_cluster.yaml

@@ -17,6 +17,9 @@ nodes:
   - containerPort: 80 # ingress controller
     hostPort: 5080
     protocol: TCP
+  - containerPort: 443
+    hostPort: 5443
+    protocol: TCP
   - containerPort: 30010 # grpc nodeport
     hostPort: 8082
     protocol: TCP
@@ -29,11 +32,9 @@ nodes:
   - containerPort: 30013 # grpc router nodeport (replica 2)
     hostPort: 8085
     protocol: TCP
-  - containerPort: 443
-    hostPort: 5443
-    protocol: TCP
+
 # if we needed to mount a hostPath volume into the kind cluster, we can do it like this
 #  extraMounts:
 #  - hostPath: ./bin/e2e-certs
 #    containerPath: /tmp/e2e-certs
-  
+

hack/utils

@@ -68,8 +68,7 @@ install_nginx_ingress() {
   lsmod | grep ip_tables || \
     (echo "ip_tables module not loaded needed by nginx ingress, please run 'sudo modprobe ip_tables'" && exit 1)
 
-  # Deploy nginx ingress for kind
-  kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/kind/deploy.yaml
+kubectl apply -f https://github.com/kubernetes/ingress-nginx/raw/refs/heads/main/deploy/static/provider/kind/deploy.yaml
 
   echo -e "${GREEN}Waiting for nginx to be ready ...${NC}"