diff --git a/Makefile.local b/Makefile.local index 74c168ec..c7d59e3e 100644 --- a/Makefile.local +++ b/Makefile.local @@ -106,6 +106,11 @@ deploy-multitool: deploy-external-dns: helm upgrade -i --repo https://kubernetes-sigs.github.io/external-dns external-dns external-dns --version 1.12.2 --values test-data/external-dns-values.yaml +################# +.PHONY: deploy-metrics-server +deploy-metrics-server: + helm upgrade -i --repo https://kubernetes-sigs.github.io/metrics-server metrics-server metrics-server --version 3.8.3 -n kube-system --set args={--kubelet-insecure-tls} + ################# # https://kind.sigs.k8s.io/docs/user/loadbalancer/ .PHONY: deploy-metallb @@ -246,6 +251,7 @@ setup-getting-started: setup-getting-started-cluster setup-getting-started-contr setup-getting-started-cluster: make create-cluster deploy-gateway-api make deploy-metallb + make deploy-metrics-server make deploy-istio make deploy-contour deploy-contour-provisioner make setup-external-dns-test diff --git a/blueprints/aws-alb-crossplane/README.md b/blueprints/aws-alb-crossplane/README.md index 04224a47..f7a706cf 100644 --- a/blueprints/aws-alb-crossplane/README.md +++ b/blueprints/aws-alb-crossplane/README.md @@ -6,7 +6,8 @@ infrastructure: - Application load balancer (ALB). - Security group for ALB, together with ingress and egress rules (for both data and healthchecks). -- ALB target group and listener definitions. +- ALB target group +- ALB listener definitions for both terminating TLS (port 443) and redirecting HTTP (port 80) to HTTPS. This definition also includes the following Kubernetes infrastructure: @@ -17,6 +18,8 @@ This definition also includes the following Kubernetes infrastructure: for propagating Kubernetes endpoints for the Istio ingress gateway to the AWS ALB target group. This links the Kubernetes internal and AWS infrastructure. +- Optional HorizontalPodAutoscaler +- Optional PodDisruptionBudget **Note** the ALB terminates TLS and forwards traffic un-encrypted to the Istio ingress gateway. diff --git a/blueprints/aws-alb-crossplane/gatewayclassblueprint-aws-alb-crossplane.yaml b/blueprints/aws-alb-crossplane/gatewayclassblueprint-aws-alb-crossplane.yaml index 27467f3d..9c876b4e 100644 --- a/blueprints/aws-alb-crossplane/gatewayclassblueprint-aws-alb-crossplane.yaml +++ b/blueprints/aws-alb-crossplane/gatewayclassblueprint-aws-alb-crossplane.yaml @@ -11,9 +11,16 @@ spec: threshold: 2 path: /healthz/ready port: 15021 + hpa: + minReplicas: 1 # optional + maxReplicas: 3 # Optional, will default to minReplicas if minReplicas is defined + averageUtilization: 60 ingressAcls: cidrs: - 0.0.0.0/0 + pdb: + minAvailable: "1" + maxUnavailable: tags: [] # Values required by this blueprint without defaults: # providerConfigName: "example-crossplane-provider-name" @@ -176,7 +183,9 @@ spec: namespace: {{ .Gateway.metadata.namespace }} {{ if .Values.tags }} annotations: - {{- toYaml .Values.tags | nindent 4 }} + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} {{ end }} spec: targetGroupARN: {{ (index .Resources.LBTargetGroup 0).status.atProvider.arn }} @@ -337,6 +346,67 @@ spec: tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} toPort: 15021 type: ingress + hpa: | + {{ if or (get .Values.hpa "minReplicas") (get .Values.hpa "maxReplicas") }} + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + labels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} + spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Gateway.metadata.name }}-child-istio + {{ if get .Values.hpa "minReplicas" }} + minReplicas: {{ .Values.hpa.minReplicas }} + {{ end }} + {{ if get .Values.hpa "maxReplicas" }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + {{ else }} # Ensure that max >= min, even when maxReplicas is left undefined + {{ if get .Values.hpa "minReplicas" }} + maxReplicas: {{ .Values.hpa.minReplicas }} + {{ end }} + {{ end }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.hpa.averageUtilization }} + {{ end }} + pdb: | + {{ if or (get .Values.pdb "minAvailable") (get .Values.pdb "maxUnavailable") }} + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} + spec: + {{ if get .Values.pdb "minAvailable" }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{ else }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{ end }} + selector: + # Match the generated Deployment by label + matchLabels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + istio.io/gateway-name: {{ .Gateway.metadata.name }}-child + {{ end }} # The following are templates used to 'implement' a 'parent' HTTPRoute httpRouteTemplate: diff --git a/blueprints/contour-istio/gatewayclassblueprint-contour-istio-cert.yaml b/blueprints/contour-istio/gatewayclassblueprint-contour-istio-cert.yaml index 5fcaff71..6570e6dc 100644 --- a/blueprints/contour-istio/gatewayclassblueprint-contour-istio-cert.yaml +++ b/blueprints/contour-istio/gatewayclassblueprint-contour-istio-cert.yaml @@ -3,6 +3,16 @@ kind: GatewayClassBlueprint metadata: name: contour-istio-cert spec: + values: + default: + hpa: + minReplicas: 1 # optional + maxReplicas: 3 # Optional, will default to minReplicas if minReplicas is defined + averageUtilization: 60 + pdb: + minAvailable: "1" + maxUnavailable: + tags: [] # The following are templates used to 'implement' a 'parent' Gateway gatewayTemplate: @@ -22,6 +32,9 @@ spec: namespace: {{ .Gateway.metadata.namespace }} annotations: networking.istio.io/service-type: ClusterIP + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: gatewayClassName: istio listeners: @@ -43,6 +56,10 @@ spec: metadata: name: {{ .Gateway.metadata.name }} namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: ingressClassName: contour tls: @@ -64,12 +81,77 @@ spec: port: number: 80 {{- end }} + hpa: | + {{ if or (get .Values.hpa "minReplicas") (get .Values.hpa "maxReplicas") }} + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + labels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} + spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Gateway.metadata.name }}-child-istio + {{ if get .Values.hpa "minReplicas" }} + minReplicas: {{ .Values.hpa.minReplicas }} + {{ end }} + {{ if get .Values.hpa "maxReplicas" }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + {{ else }} # Ensure that max >= min, even when maxReplicas is left undefined + {{ if get .Values.hpa "minReplicas" }} + maxReplicas: {{ .Values.hpa.minReplicas }} + {{ end }} + {{ end }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.hpa.averageUtilization }} + {{ end }} + pdb: | + {{ if or (get .Values.pdb "minAvailable") (get .Values.pdb "maxUnavailable") }} + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} + spec: + {{ if get .Values.pdb "minAvailable" }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{ else }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{ end }} + selector: + # Match the generated Deployment by label + matchLabels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + istio.io/gateway-name: {{ .Gateway.metadata.name }}-child + {{ end }} tlsCertificate: | apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: {{ .Gateway.metadata.name }}-cert namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: secretName: {{ .Gateway.metadata.name }}-tls duration: 2160h # 90d @@ -103,6 +185,10 @@ spec: metadata: name: {{ .HTTPRoute.metadata.name }}-child namespace: {{ .HTTPRoute.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: parentRefs: {{ range .HTTPRoute.spec.parentRefs }} diff --git a/blueprints/contour-istio/gatewayclassblueprint-contour-istio.yaml b/blueprints/contour-istio/gatewayclassblueprint-contour-istio.yaml index 48e829e2..0349dc32 100644 --- a/blueprints/contour-istio/gatewayclassblueprint-contour-istio.yaml +++ b/blueprints/contour-istio/gatewayclassblueprint-contour-istio.yaml @@ -3,6 +3,16 @@ kind: GatewayClassBlueprint metadata: name: contour-istio spec: + values: + default: + hpa: + minReplicas: 1 # optional + maxReplicas: 3 # Optional, will default to minReplicas if minReplicas is defined + averageUtilization: 60 + pdb: + minAvailable: "1" + maxUnavailable: + tags: [] # The following are templates used to 'implement' a 'parent' Gateway gatewayTemplate: @@ -15,6 +25,9 @@ spec: namespace: {{ .Gateway.metadata.namespace }} annotations: networking.istio.io/service-type: ClusterIP + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: gatewayClassName: istio listeners: @@ -36,6 +49,10 @@ spec: metadata: name: {{ .Gateway.metadata.name }} namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: ingressClassName: contour tls: @@ -57,6 +74,67 @@ spec: port: number: 80 {{- end }} + hpa: | + {{ if or (get .Values.hpa "minReplicas") (get .Values.hpa "maxReplicas") }} + apiVersion: autoscaling/v2 + kind: HorizontalPodAutoscaler + metadata: + labels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} + spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ .Gateway.metadata.name }}-child-istio + {{ if get .Values.hpa "minReplicas" }} + minReplicas: {{ .Values.hpa.minReplicas }} + {{ end }} + {{ if get .Values.hpa "maxReplicas" }} + maxReplicas: {{ .Values.hpa.maxReplicas }} + {{ else }} # Ensure that max >= min, even when maxReplicas is left undefined + {{ if get .Values.hpa "minReplicas" }} + maxReplicas: {{ .Values.hpa.minReplicas }} + {{ end }} + {{ end }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.hpa.averageUtilization }} + {{ end }} + pdb: | + {{ if or (get .Values.pdb "minAvailable") (get .Values.pdb "maxUnavailable") }} + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + namespace: {{ .Gateway.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} + spec: + {{ if get .Values.pdb "minAvailable" }} + minAvailable: {{ .Values.pdb.minAvailable }} + {{ else }} + maxUnavailable: {{ .Values.pdb.maxUnavailable }} + {{ end }} + selector: + # Match the generated Deployment by label + matchLabels: + tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }} + istio.io/gateway-name: {{ .Gateway.metadata.name }}-child + {{ end }} # The following are templates used to 'implement' a 'parent' HTTPRoute httpRouteTemplate: @@ -67,6 +145,10 @@ spec: metadata: name: {{ .HTTPRoute.metadata.name }}-child namespace: {{ .HTTPRoute.metadata.namespace }} + annotations: + {{ if .Values.tags }} + {{ toYaml .Values.tags | nindent 4 }} + {{ end }} spec: parentRefs: {{ range .HTTPRoute.spec.parentRefs }} diff --git a/charts/bifrost-gateway-controller/CHANGELOG.md b/charts/bifrost-gateway-controller/CHANGELOG.md index 6537d9df..75288012 100644 --- a/charts/bifrost-gateway-controller/CHANGELOG.md +++ b/charts/bifrost-gateway-controller/CHANGELOG.md @@ -2,7 +2,11 @@ ## [UNRELEASED] -- Example text, add your PR info according to example below below this line. Do not bump chart version in Chart.yaml. +- Example text, add your PR info according to example below below this line. Do not bump chart version in Chart.yaml unless a chart release will be made following your PR. + +## [0.1.7] + +- Add HorizontalPodAutoscaler and PodDisruptionBudget resources to aws-crossplane blueprint and update Helm chart example values with RBAC for HPA and PDB. ([#186](https://github.com/tv2-oss/bifrost-gateway-controller/pull/186)) [@michaelvl](https://github.com/michaelvl) ## [0.1.8] diff --git a/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml b/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml index 1c157b84..86044b50 100644 --- a/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml +++ b/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml @@ -26,3 +26,27 @@ controllerManager: - patch - update - watch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml b/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml index 82305940..065df3b0 100644 --- a/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml +++ b/charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml @@ -41,3 +41,27 @@ controllerManager: - patch - update - watch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/config/kind/kustomization.yaml b/config/kind/kustomization.yaml index a6f3479d..1bf629d0 100644 --- a/config/kind/kustomization.yaml +++ b/config/kind/kustomization.yaml @@ -58,6 +58,30 @@ patchesStrategicMerge: - patch - update - watch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch # Below are copied from role.yaml - apiGroups: - gateway.networking.k8s.io diff --git a/hack/demo/show-resources.sh b/hack/demo/show-resources.sh index ce4ebe49..9c34ce04 100755 --- a/hack/demo/show-resources.sh +++ b/hack/demo/show-resources.sh @@ -1,3 +1,3 @@ #! /bin/bash -kubectl get gateway,lbs,lbtargetgroups,lblisteners,securitygroups,securitygrouprules,targetgroupbindings -A | sed -E 's#(arn:aws:elasticloadbalancing:eu-central-1:)[0-9]+(:[-0-9a-z\/]+)#\11234567890\2#' +kubectl get gateway,lbs,lbtargetgroups,lblisteners,securitygroups,securitygrouprules,targetgroupbindings,hpa,pdb -A | sed -E 's#(arn:aws:elasticloadbalancing:eu-central-1:)[0-9]+(:[-0-9a-z\/]+)#\11234567890\2#'