GKE script that allows overrides

bin/y-cluster-provision-gke

@@ -4,6 +4,7 @@ set -e
 
 NAME="$1"
 [ -z "$NAME" ] && echo "First argument must be a cluster name" && exit 1
+shift 1
 
 FLAGS=""
 
@@ -18,13 +19,14 @@ gcloud container get-server-config --region=$REGION | grep "1.14"
 
 FLAGS="$FLAGS --cluster-version=1.14.6-gke.1"
 FLAGS="$FLAGS --image-type=UBUNTU_CONTAINERD"
-FLAGS="$FLAGS --machine-type=g1-small"
+
 FLAGS="$FLAGS --num-nodes=1"
+FLAGS="$FLAGS --enable-autoscaling"
+FLAGS="$FLAGS --max-nodes=10"
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster#create_regional
-#FLAGS="$FLAGS --region=$REGION"
-# Temporary, non-regional
-FLAGS="$FLAGS --zone=$REGION-a"
+FLAGS="$FLAGS --region=$REGION"
+# Use https://cloud.google.com/sdk/gcloud/reference/container/clusters/update#--node-locations to add zones, and notee that num-nodes is multiplied
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/small-cluster-tuning
 #FLAGS="$FLAGS --logging-service none --monitoring-service none"
@@ -34,6 +36,7 @@ FLAGS="$FLAGS --no-enable-stackdriver-kubernetes"
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades
 #FLAGS="$FLAGS --enable-autoupgrade"
+# TODO maintenance window
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/container-native-load-balancing
 NETWORK=default
@@ -79,7 +82,7 @@ echo "### What this script will try to do is ..."
 echo "Use the currently configured project: $PROJECT"
 echo "Use the currently configured region: $REGION"
 echo "Add authentication to KUBECONFIG=$KUBECONFIG"
-echo "Create a GKE cluster using the following flags: $FLAGS"
+echo "Create a GKE cluster using the following flags: $FLAGS $@"
 echo ""
 
 read -p 'Please confirm [y/n]? ' yn
@@ -88,8 +91,7 @@ case $yn in
   * ) exit;;
 esac
 
-#gcloud container clusters create "$NAME" $FLAGS
-gcloud container clusters create "$NAME" --create-subnetwork "" $FLAGS
+gcloud container clusters create "$NAME" $FLAGS "$@"
 
 GKE_CONTEXT_NAME=$(kubectl config current-context)
 
@@ -105,27 +107,31 @@ echo gcloud compute routers nats create nat-config \
     --auto-allocate-nat-external-ips
 
 echo "To clean up:"
-#echo gcloud container clusters delete --region $REGION $NAME
-echo gcloud container clusters delete --zone $REGION-a $NAME
+echo gcloud container clusters delete --zone $REGION $NAME
 echo gcloud compute routers nats delete --router-region $REGION --router nat-router nat-config
 echo gcloud compute routers delete nat-router --region $REGION
 
+kubectl create namespace ystack
+
+kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
+
 ### To be moved to registry provision script, but we do want the initial context name
 
+[ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
+
+kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
+
 gcloud compute backend-buckets list
 gsutil ls
 # https://cloud.google.com/config-connector/docs/overview
-mkdir -p gke/registry
-cat << EOF > gke/registry/builds-gcs-storagebucket.yaml
+cat << EOF > $YSTACK_HOME/gke/registry/builds-gcs-storagebucket.yaml
 # https://github.com/GoogleCloudPlatform/k8s-config-connector/blob/master/resources/storagebucket/storage_v1alpha2_storagebucket.yaml
 apiVersion: storage.cnrm.cloud.google.com/v1alpha2
 kind: StorageBucket
 metadata:
-  name: ${GKE_CONTEXT_NAME}_builds
+  name: ${BUILDS_BUCKET_NAME}
 spec: {}
 EOF
 
-# TODO we also need to generate a bucket name, or a secret with a bucket name, for gke/registry/builds-gcs-bucket-storage.yaml
-
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
-  gsutil mb -c regional -l $REGION gs://${GKE_CONTEXT_NAME}_builds
+  gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME}

converge-gke/kustomization.yaml

@@ -0,0 +1,10 @@
+namespace: ystack
+bases:
+- ../gke/storageclasses
+- ../buildkit
+- ../gke/registry
+- ../registry/node-update-hosts
+# Two ways to get nodes' containerd to allow pull from in-cluster .local service
+# (assuming we already have DNS name lookup fixed through for example ../registry/hosts-update):
+#- ../../registry/generic,tls/       # requires manual containerd restart until https://github.com/containerd/containerd/issues/3071 is resolved
+- ../registry/node-update-containerd # also requires restart but needs no TLS support in registry so might be a simpler option until we have containerd#3071

examples/basic-dev-inner-loop/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:10.15.3-alpine
+FROM node:10.16.3-alpine@sha256:77c898d0da5e7bfb6e05c9a64de136ba4e03889a72f3c298e95df822a38f450d
 
 WORKDIR /app
 EXPOSE 3000

gke/registry/builds-gcs-bucket-storage.yaml

@@ -7,9 +7,11 @@ spec:
     spec:
       containers:
       - name: docker-v2
-        image: registry:2.7.1@sha256:a54bc9be148764891c44676ce8c44f1e53514c43b1bfbab87b896f4b9f0b5d99
         env:
         - name: REGISTRY_STORAGE
           value: gcs
         - name: REGISTRY_STORAGE_GCS_BUCKET
-          value: # TODO must be generated by the provision script
+          valueFrom:
+            secretKeyRef:
+              name: registry-persistence
+              key: gcsBucketName

gke/registry/kustomization.yaml

@@ -1,8 +1,4 @@
 bases:
 - ../../registry/generic/
-# Two ways to get nodes' containerd to allow pull from in-cluster .local service
-# (assuming we already have DNS name lookup fixed through for example ../registry/hosts-update):
-#- ../../registry/generic,tls/       # requires manual containerd restart until https://github.com/containerd/containerd/issues/3071 is resolved
-- ../../registry/node-update-containerd/ # also requires restart but needs no TLS support in registry so might be a simpler option until we have containerd#3071
 patchesStrategicMerge:
 - builds-gcs-bucket-storage.yaml

gke/storageclasses/default.yaml

@@ -0,0 +1,6 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: zonal-ssd-resizable-delete
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"

gke/storageclasses/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+- standard-non-default.yaml
+- zonal-ssd-resizable-retain.yaml
+- zonal-ssd-resizable-delete.yaml
+patchesStrategicMerge:
+- default.yaml

gke/storageclasses/standard-non-default.yaml

@@ -0,0 +1,10 @@
+# based on a fresh GKE cluster's kubectl get storageclass standard -o yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-standard

gke/storageclasses/zonal-ssd-resizable-delete.yaml

@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: zonal-ssd-resizable-delete
+provisioner: kubernetes.io/gce-pd
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+parameters:
+  type: pd-ssd

gke/storageclasses/zonal-ssd-resizable-retain.yaml

@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: zonal-ssd-resizable-retain
+provisioner: kubernetes.io/gce-pd
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+parameters:
+  type: pd-ssd

registry/node-update-containerd/daemonset.yaml

@@ -11,26 +11,46 @@ spec:
       labels:
         app: containerd-registry-http
     spec:
+      hostPID: true
+      hostNetwork: true
       initContainers:
       - name: update
         image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
         volumeMounts:
         - name: etc-containerd
           mountPath: /etc/containerd
           readOnly: false
+        - name: results
+          mountPath: /results
+        env:
+        - name: CONFIG_PATH
+          value: /etc/containerd/config.toml
+        securityContext:
+          privileged: true
         command:
         - bash
         #to troubleshoot use: - -cex
         - -ce
         - |
-          cat << EOF >> /etc/containerd/config.toml
-          [plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]
-             endpoint = ["http://builds-registry.ystack.svc.cluster.local"]
-          [plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]
-             endpoint = ["http://prod-registry.ystack.svc.cluster.local"]
-          EOF
-          echo "TODO currently you must manually ssh to the node and run: systemctl restart containerd"
-          # maybe https://alexei-led.github.io/post/k8s_node_shell/ ?
+          if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then
+            echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH
+            echo '  endpoint = ["http://builds-registry.ystack.svc.cluster.local"]'          >> $CONFIG_PATH
+            touch /results/config-updated
+          fi
+          if ! grep prod-registry.ystack.svc.cluster.local $CONFIG_PATH; then
+            echo '[plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]'   >> $CONFIG_PATH
+            echo '  endpoint = ["http://prod-registry.ystack.svc.cluster.local"]'            >> $CONFIG_PATH
+            touch /results/config-updated
+          fi
+          cat $CONFIG_PATH
+           if [ -f /results/config-updated ]; then
+            echo "Configuration has been updated; restarting containerd (not expecting any output) ..."
+            nsenter --mount=/proc/1/ns/mnt -- systemctl restart containerd
+            echo "Restart triggered"
+           else
+            echo "No configration change"
+           fi
+          echo "Done"
       containers:
       - name: init-container-did-the-work
         image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610
@@ -39,3 +59,5 @@ spec:
       - name: etc-containerd
         hostPath:
           path: /etc/containerd
+      - name: results
+        emptyDir: {}

registry/node-update-hosts/daemonset.yaml

@@ -23,7 +23,6 @@ spec:
           value: >-
             builds-registry.ystack.svc.cluster.local
             prod-registry.ystack.svc.cluster.local
-            blobs-minio.ystack.svc.cluster.local
         command:
         - bash
         - -ce