Test AWS/crossplane blueprint against different crossplane aws provider versions

.github/dependabot.yml

@@ -8,7 +8,7 @@ updates:
       # The version of client-go and other Kubernetes APIs should approximately match target Kubernetes version, i.e. only update semver-patch version
       # Minor version updates then becomes a manual procedure. Security updates are not ignored by this
       - dependency-name: "k8s.io/*"
-        versions: ["version-update:semver-minor"]
+        update-types: ["version-update:semver-minor"]
 
   - package-ecosystem: "github-actions"
     directory: "/"

Makefile.local

@@ -43,9 +43,7 @@ wait-ready-external-dns-test:
 	until kubectl wait pods -l app.kubernetes.io/instance=external-dns --for condition=Ready --timeout=120s     ; do echo "."; sleep 1; done
 
 #################
-ifeq ($(GATEWAY_API_VERSION),)
-GATEWAY_API_VERSION=v0.6.0
-endif
+GATEWAY_API_VERSION ?= v0.6.0
 
 .PHONY: gateway-api-upstream-get
 gateway-api-upstream-get:
@@ -72,10 +70,16 @@ delete-cluster:
 	kind delete cluster --name kind-gwc-dev-cluster
 
 #################
+ISTIO_VERSION ?= 1.16.1
+
 .PHONY: deploy-istio
 deploy-istio:
-	helm upgrade -i --repo https://istio-release.storage.googleapis.com/charts base base     --version 1.16.1 -n istio-system --create-namespace
-	helm upgrade -i --repo https://istio-release.storage.googleapis.com/charts istiod istiod --version 1.16.1 -n istio-system
+	helm upgrade -i --repo https://istio-release.storage.googleapis.com/charts base base     --version ${ISTIO_VERSION} -n istio-system --create-namespace
+	helm upgrade -i --repo https://istio-release.storage.googleapis.com/charts istiod istiod --version ${ISTIO_VERSION} -n istio-system
+
+.PHONY: undeploy-istio
+undeploy-istio:
+	helm uninstall -n istio-system istiod
 
 #################
 .PHONY: cluster-load-controller-image
@@ -143,25 +147,100 @@ ca-cert-secret-create:
 	kubectl -n cert-manager create secret tls ca-key-pair --cert=foo-example-com.crt --key=foo-example-com.key
 
 #################
+AWS_LOAD_BALANCER_CONTROLLER_VERSION ?= v2.4.5
+AWS_LOAD_BALANCER_CONTROLLER_CHART_VERSION ?= v1.4.6
+# Note, template also require CLUSTERNAME and AWS_LOAD_BALANCER_CONTROLLER_IAM_ROLE_ARN
+
 .PHONY: deploy-aws-load-balancer-controller-crds
 deploy-aws-load-balancer-controller-crds:
-	kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.4.5/helm/aws-load-balancer-controller/crds/crds.yaml
+	kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/${AWS_LOAD_BALANCER_CONTROLLER_VERSION}/helm/aws-load-balancer-controller/crds/crds.yaml
+
+.PHONY: deploy-aws-load-balancer-controller
+deploy-aws-load-balancer-controller:
+	cat test-data/aws-load-balancer-controller-values.yaml_tpl | envsubst > aws-load-balancer-controller-values.yaml
+	helm upgrade -i --repo https://aws.github.io/eks-charts aws-load-balancer-controller aws-load-balancer-controller --version ${AWS_LOAD_BALANCER_CONTROLLER_CHART_VERSION} -n kube-system --set installCRDs=false --values aws-load-balancer-controller-values.yaml
+
+.PHONY: undeploy-aws-load-balancer-controller
+undeploy-aws-load-balancer-controller:
+	helm uninstall -n kube-system aws-load-balancer-controller
 
 #################
+CROSSPLANE_VERSION ?= v1.11.0
+
 .PHONY: deploy-crossplane
 deploy-crossplane:
-	helm upgrade -i --repo https://charts.crossplane.io/stable crossplane crossplane --version v1.11.0 -n crossplane-system --create-namespace
+	helm upgrade -i --repo https://charts.crossplane.io/stable crossplane crossplane --version ${CROSSPLANE_VERSION} -n crossplane-system --create-namespace
+
+.PHONY: undeploy-crossplane
+undeploy-crossplane:
+	helm uninstall crossplane -n crossplane-system
+
+CROSSPLANE_AWS_PROVIDER_VERSION ?= v0.28.0
+# Note, templates also require CROSSPLANE_INITIAL_IAM_ROLE_ARN and CROSSPLANE_IAM_ROLE_ARN
 
 .PHONY: deploy-crossplane-aws-provider
 deploy-crossplane-aws-provider:
-	kubectl apply -f test-data/crossplane-aws-provider.yaml
+	cat test-data/crossplane-aws-provider.yaml_tpl | CROSSPLANE_AWS_PROVIDER_VERSION=${CROSSPLANE_AWS_PROVIDER_VERSION} CROSSPLANE_INITIAL_IAM_ROLE_ARN=${CROSSPLANE_INITIAL_IAM_ROLE_ARN} envsubst > crossplane-aws-provider.yaml
+	cat test-data/crossplane-aws-provider-config.yaml_tpl | CROSSPLANE_IAM_ROLE_ARN=${CROSSPLANE_IAM_ROLE_ARN} envsubst > crossplane-aws-provider-config.yaml
+	kubectl apply -f crossplane-aws-provider.yaml
 	kubectl wait "providers.pkg.crossplane.io/provider-aws" --for=condition=Installed --timeout=180s
 	kubectl wait "providers.pkg.crossplane.io/provider-aws" --for=condition=Healthy --timeout=180s
+	kubectl apply -f crossplane-aws-provider-config.yaml
+
+.PHONY: undeploy-crossplane-aws-provider
+undeploy-crossplane-aws-provider:
+	kubectl delete -f crossplane-aws-provider-config.yaml
+	kubectl delete -f crossplane-aws-provider.yaml
+
+#################
+BIFROST_VERSION ?= 0.1.6
+
+.PHONY: deploy-controller-helm
+deploy-controller-helm:
+	helm upgrade -i bifrost-gateway-controller-helm oci://ghcr.io/tv2-oss/bifrost-gateway-controller-helm --version ${BIFROST_VERSION} --values charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml -n bifrost-gateway-controller-system --create-namespace
+
+.PHONY: deploy-controller-aws-helm
+deploy-controller-aws-helm:
+	helm upgrade -i bifrost-gateway-controller-helm oci://ghcr.io/tv2-oss/bifrost-gateway-controller-helm --version ${BIFROST_VERSION} --values charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml -n bifrost-gateway-controller-system --create-namespace
+
+.PHONY: undeploy-controller
+undeploy-controller:
+	helm uninstall -n bifrost-gateway-controller-system bifrost-gateway-controller-helm
+
+#################
+BIFROST_BLUEPRINTS_VERSION ?= 0.0.18
+
+.PHONY: deploy-controller-blueprint
+setup-getting-started-controller-blueprint:
+	kubectl apply -f https://github.com/tv2-oss/bifrost-gateway-controller/releases/download/${BIFROST_BLUEPRINTS_VERSION}/gatewayclassblueprint-contour-istio-cert.yaml
+	kubectl apply -f https://github.com/tv2-oss/bifrost-gateway-controller/releases/download/${BIFROST_BLUEPRINTS_VERSION}/gatewayclass-contour-istio-cert.yaml
+
+.PHONY: deploy-controller-blueprint-local
+deploy-controller-blueprint-local:
+	kubectl apply -f blueprints/gatewayclassblueprint-contour-istio-cert.yaml -f blueprints/gatewayclass-contour-istio-cert.yaml
+
+.PHONY: deploy-aws-istio-blueprint
+deploy-aws-istio-blueprint:
+	kubectl apply -f https://github.com/tv2-oss/bifrost-gateway-controller/releases/download/${BIFROST_BLUEPRINTS_VERSION}/gatewayclassblueprint-aws-alb-crossplane.yaml
+	kubectl apply -f https://github.com/tv2-oss/bifrost-gateway-controller/releases/download/${BIFROST_BLUEPRINTS_VERSION}/gatewayclass-aws-alb-crossplane.yaml
+
+.PHONY: undeploy-aws-istio-blueprint
+undeploy-aws-istio-blueprint:
+	kubectl delete -f https://github.com/tv2-oss/bifrost-gateway-controller/releases/download/${BIFROST_BLUEPRINTS_VERSION}/gatewayclassblueprint-aws-alb-crossplane.yaml
+	kubectl delete -f https://github.com/tv2-oss/bifrost-gateway-controller/releases/download/${BIFROST_BLUEPRINTS_VERSION}/gatewayclass-aws-alb-crossplane.yaml
+
+.PHONY: deploy-aws-istio-blueprint-local
+deploy-aws-istio-blueprint-local:
+	kubectl apply -f blueprints/gatewayclassblueprint-aws-alb-crossplane.yaml -f blueprints/gatewayclass-aws-alb-crossplane.yaml
+
+.PHONY: undeploy-aws-istio-blueprint-local
+undeploy-aws-istio-blueprint-local:
+	kubectl delete -f blueprints/gatewayclassblueprint-aws-alb-crossplane.yaml -f blueprints/gatewayclass-aws-alb-crossplane.yaml
 
 #################
 # See 'doc/getting-started.md'
 .PHONY: setup-getting-started
-setup-getting-started: setup-getting-started-cluster setup-getting-started-controller setup-getting-started-usecase
+setup-getting-started: setup-getting-started-cluster setup-getting-started-controller setup-getting-started-controller-blueprint deploy-getting-started-usecase
 
 .PHONY: setup-getting-started-cluster
 setup-getting-started-cluster:
@@ -179,23 +258,29 @@ setup-getting-started-controller:
 	make docker-build
 	make cluster-load-controller-image
 	make deploy
-	kubectl apply -f blueprints/gatewayclassblueprint-contour-istio-cert.yaml -f blueprints/gatewayclass-contour-istio-cert.yaml
 
-.PHONY: setup-getting-started-controller-helm
-setup-getting-started-controller-helm:
-	helm upgrade -i bifrost-gateway-controller-helm oci://ghcr.io/tv2-oss/bifrost-gateway-controller-helm --version 0.1.6 --values charts/bifrost-gateway-controller/ci/gatewayclassblueprint-contour-istio-values.yaml -n bifrost-gateway-controller-system --create-namespace
-	kubectl apply -f blueprints/gatewayclassblueprint-contour-istio-cert.yaml -f blueprints/gatewayclass-contour-istio-cert.yaml
+GATEWAY_CLASS_NAME ?= contour-istio-cert
+DOMAIN ?= foo.example.com
 
-.PHONY: setup-getting-started-usecase
-setup-getting-started-usecase:
+.PHONY: deploy-getting-started-usecase
+deploy-getting-started-usecase:
 	kubectl apply -f test-data/getting-started/foo-namespaces.yaml
-	kubectl apply -f test-data/getting-started/foo-gateway.yaml
+	cat test-data/getting-started/foo-gateway.yaml | GATEWAY_CLASS_NAME=${GATEWAY_CLASS_NAME} DOMAIN=${DOMAIN} envsubst | kubectl apply -f -
 	kubectl -n foo-site apply -f test-data/getting-started/app-foo-site.yaml
 	kubectl -n foo-site apply -f test-data/getting-started/foo-site-httproute.yaml
 	kubectl -n foo-store apply -f test-data/getting-started/app-foo-store-v1.yaml
 	kubectl -n foo-store apply -f test-data/getting-started/app-foo-store-v2.yaml
 	kubectl -n foo-store apply -f test-data/getting-started/foo-store-httproute.yaml
 
+.PHONY: undeploy-getting-started-usecase
+undeploy-getting-started-usecase:
+	kubectl delete -f test-data/getting-started/foo-namespaces.yaml
+
+.PHONY: deploy-namespace-gatewayclassconfig
+deploy-namespace-gatewayclassconfig:
+	kubectl apply -f test-data/getting-started/foo-namespaces.yaml
+	cat hack/demo/namespace-gatewayclassconfig.yaml | CERTIFICATE_ARN=${CERTIFICATE_ARN} envsubst | kubectl apply -f -
+
 .PHONY: wait-ready-getting-started-usecase
 wait-ready-getting-started-usecase:
 	scripts/waitfor.sh scripts/curl.sh -s --connect-timeout 1 --fail --resolve foo.example.com:80:127.0.0.1 http://foo.example.com/site

blueprints/README.md

@@ -55,5 +55,19 @@ This definition is provided in the following files:
 - [`gatewayclassblueprint-aws-alb-crossplane.yaml`](gatewayclassblueprint-aws-alb-crossplane.yaml) blueprint for infrastructure implementation
 - [`gatewayclass-aws-alb-crossplane.yaml`](gatewayclass-aws-alb-crossplane.yaml) definitions of `GatewayClass`es referencing the above `GatewayClassBlueprint`. Two `GatewayClass`es are created, one that is intended for internet exposed gateways, and one for non internet exposed gateways.
 - [`gatewayclassconfig-aws-alb-crossplane-dev-env.yaml`](../test-data/gatewayclassconfig-aws-alb-crossplane-dev-env.yaml) example settings for the two `GatewayClass`es defined in `gatewayclass-aws-alb-crossplane.yaml`, i.e. with different subnet settings for the internet-exposed and non internet-exposed `GatewayClass'es.
-[`gatewayclassblueprint-crossplane-aws-alb-values.yaml`](../charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml)
-(RBAC for bifrost-gateway-controller Helm deployment suited for the `aws-alb-crossplane` blueprint).
+- [`gatewayclassblueprint-crossplane-aws-alb-values.yaml`](../charts/bifrost-gateway-controller/ci/gatewayclassblueprint-crossplane-aws-alb-values.yaml)
+RBAC for bifrost-gateway-controller Helm deployment suited for the `aws-alb-crossplane` blueprint.
+
+### Compatibility
+
+This blueprint use AWS Crossplane resources through the [Upbound AWS
+Provider](https://marketplace.upbound.io/providers/upbound/provider-aws). The
+following compatibility between this blueprint, Crossplane, Crossplane
+Upbound AWS provider and Istio versions has been verified:
+
+| Blueprint | AWS Provider | Crossplane | Istio | Status |
+| ------------- | ------------- |
+| `0.0.18` | `v0.28.0` | `v1.11.0` | `1.16.1` | :heavy_check_mark: |
+| `0.0.18` | `v0.32.1` | `v1.11.0` | `1.16.1` | :x: |
+| `0.0.18` | `v0.33.0` | `v1.11.0` | `1.16.1` | :heavy_check_mark: |
+| `0.0.19` | `v0.33.0` | `v1.11.0` | `1.16.1` | :heavy_check_mark: |

blueprints/gatewayclassblueprint-aws-alb-crossplane.yaml

@@ -17,6 +17,7 @@ spec:
         port: 443
       tags: []
     # Values required by this blueprint without defaults:
+    #   providerConfigName: "example-crossplane-provider-name"
     #   region: "example-region"
     #   vpcId:  "example-vpc"
     #   subnets:
@@ -42,7 +43,9 @@ spec:
           namespace: {{ .Gateway.metadata.namespace }}
           annotations:
             networking.istio.io/service-type: ClusterIP
+            {{ if .Values.tags }}
             {{ toYaml .Values.tags | nindent 4 }}
+            {{ end }}
         spec:
           gatewayClassName: istio
           listeners:
@@ -67,7 +70,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
             region: {{ .Values.region }}
@@ -79,8 +82,10 @@ spec:
               {{ range .Values.subnets }}
               - subnetId: {{ . }}
               {{ end }}
+            {{ if .Values.tags }}
             tags:
               {{- toYaml .Values.tags | nindent 6 }}
+            {{ end }}
       LBTargetGroup: |
         apiVersion: elbv2.aws.upbound.io/v1beta1
         kind: LBTargetGroup
@@ -90,7 +95,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
             region: {{ .Values.region }}
@@ -104,8 +109,10 @@ spec:
               port: {{ .Values.healthCheck.port | quote }}
             port: 80
             protocol: HTTP
+            {{ if .Values.tags }}
             tags:
               {{- toYaml .Values.tags | nindent 6 }}
+            {{ end }}
             targetType: ip
       LBListener: |
         apiVersion: elbv2.aws.upbound.io/v1beta1
@@ -116,7 +123,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             region: {{ .Values.region }}
             port: 443
@@ -130,16 +137,20 @@ spec:
             loadBalancerArnSelector:
               matchLabels:
                 tv2.dk/gw: {{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
+            {{ if .Values.tags }}
             tags:
               {{- toYaml .Values.tags | nindent 6 }}
+            {{ end }}
       TargetGroupBinding: |
         apiVersion: elbv2.k8s.aws/v1beta1
         kind: TargetGroupBinding
         metadata:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
           namespace: {{ .Gateway.metadata.namespace }}
+          {{ if .Values.tags }}
           annotations:
             {{- toYaml .Values.tags | nindent 4 }}
+          {{ end }}
         spec:
           targetGroupARN: {{ .Resources.LBTargetGroup.status.atProvider.arn }}
           targetType: ip
@@ -155,13 +166,15 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             description: "SG for ALB"
             name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}
             region: {{ .Values.region }}
+            {{ if .Values.tags }}
             tags:
               {{- toYaml .Values.tags | nindent 6 }}
+            {{ end }}
             vpcId: {{ .Values.vpcId}}
       SecurityGroupRuleEgress80: |
         apiVersion: ec2.aws.upbound.io/v1beta1
@@ -172,7 +185,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}-egress80
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             description: "Traffic towards Istio ingress gateway"
             cidrBlocks:
@@ -194,7 +207,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}-egress15021
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             description: "Healthcheck towards Istio ingress gateway"
             cidrBlocks:
@@ -216,7 +229,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}-ingress
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             description: "External traffic towards ALB"
             cidrBlocks:
@@ -240,7 +253,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}-upstream80
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             description: {{ printf "Ingress from gw-%s-%s" .Gateway.metadata.namespace .Gateway.metadata.name }}
             fromPort: 80
@@ -261,7 +274,7 @@ spec:
           name: gw-{{ .Gateway.metadata.namespace }}-{{ .Gateway.metadata.name }}-upstream15021
         spec:
           providerConfigRef:
-            name: admin
+            name: {{ .Values.providerConfigName }}
           forProvider:
             description: {{ printf "Healthcheck ingress from gw-%s-%s" .Gateway.metadata.namespace .Gateway.metadata.name }}
             fromPort: 15021
@@ -284,8 +297,10 @@ spec:
           name: {{ .HTTPRoute.metadata.name }}-child
           namespace: {{ .HTTPRoute.metadata.namespace }}
           annotations:
+            {{ if .Values.tags }}
             tags:
               {{- toYaml .Values.tags | nindent 4 }}
+            {{ end }}
         spec:
           parentRefs:
           {{ range .HTTPRoute.spec.parentRefs -}}

doc/getting-started.md

@@ -125,10 +125,11 @@ that is out out-of-scope for this guide):
 kubectl apply -f test-data/getting-started/foo-namespaces.yaml
 ```
 
-The cluster-operator/SRE also creates the common `Gateway`:
+The cluster-operator/SRE also creates the common `Gateway` using the
+`GatewayClass` created previously:
 
 ```
-kubectl apply -f test-data/getting-started/foo-gateway.yaml
+cat test-data/getting-started/foo-gateway.yaml | GATEWAY_CLASS_NAME=contour-istio-cert DOMAIN=foo.example.com envsubst | kubectl apply -f -
 ```
 
 ### Developer of 'Site' Application

doc/installing.md

@@ -39,4 +39,4 @@ helm upgrade -i bifrost-gateway-controller-helm oci://ghcr.io/tv2-oss/bifrost-ga
 
 In addition to the *bifrost-gateway-controller*, you will need
 blueprints defining datapath implementations. See [Example
-GatewayClassBlueprints](blueprints/README.md).
+GatewayClassBlueprints](../blueprints/README.md).

hack/demo/curl.sh

@@ -1,8 +1,9 @@
 #! /bin/bash
 
+DOMAIN=$1
+
 ADDR=`kubectl -n foo-infra get gateway foo-gateway -o jsonpath='{.status.addresses[0].value}'`
 IP=`dig "$ADDR" +short | head -n1`
-DOMAIN=foo.kubecon23.tv2dev.dk
 
 echo "-------------------------------------------------------------------"
 echo "Skipping DNS, using $DOMAIN = $IP"