From 7d2c382e8712b3b200f2ef21f1d342127a03f9f1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 24 Sep 2019 15:10:21 +0200
Subject: [PATCH 01/22] More issues with this command-as-env concept

---
 bin/y-cluster-provision-gke | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index b6940e20..b4786bc3 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -19,6 +19,9 @@ gcloud container get-server-config --region=$REGION | grep "1.14"
 FLAGS="$FLAGS --cluster-version=1.14.6-gke.1"
 FLAGS="$FLAGS --image-type=UBUNTU_CONTAINERD"
 FLAGS="$FLAGS --machine-type=g1-small"
+gcloud beta compute zones describe $REGION-a --format="value(availableCpuPlatforms)"
+#FLAGS="$FLAGS --min-cpu-platform 'Intel Skylake'"
+
 FLAGS="$FLAGS --num-nodes=1"
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster#create_regional
@@ -89,7 +92,10 @@ case $yn in
 esac
 
 #gcloud container clusters create "$NAME" $FLAGS
-gcloud container clusters create "$NAME" --create-subnetwork "" $FLAGS
+gcloud container clusters create "$NAME" \
+  --min-cpu-platform 'Intel Skylake' \
+  --create-subnetwork "" \
+  $FLAGS
 
 GKE_CONTEXT_NAME=$(kubectl config current-context)
 

From 71e939d43da5ce5af0a0624a84cd83e4286ab8c0 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 24 Sep 2019 15:10:51 +0200
Subject: [PATCH 02/22] Start small, let grow in nodes and zones

---
 bin/y-cluster-provision-gke | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index b4786bc3..59f1edf7 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -23,11 +23,12 @@ gcloud beta compute zones describe $REGION-a --format="value(availableCpuPlatfor
 #FLAGS="$FLAGS --min-cpu-platform 'Intel Skylake'"
 
 FLAGS="$FLAGS --num-nodes=1"
+FLAGS="$FLAGS --enable-autoscaling"
+FLAGS="$FLAGS --max-nodes=10"
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster#create_regional
-#FLAGS="$FLAGS --region=$REGION"
-# Temporary, non-regional
-FLAGS="$FLAGS --zone=$REGION-a"
+FLAGS="$FLAGS --region=$REGION"
+# Use https://cloud.google.com/sdk/gcloud/reference/container/clusters/update#--node-locations to add zones, and notee that num-nodes is multiplied
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/small-cluster-tuning
 #FLAGS="$FLAGS --logging-service none --monitoring-service none"
@@ -37,6 +38,7 @@ FLAGS="$FLAGS --no-enable-stackdriver-kubernetes"
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades
 #FLAGS="$FLAGS --enable-autoupgrade"
+# TODO maintenance window
 
 # https://cloud.google.com/kubernetes-engine/docs/how-to/container-native-load-balancing
 NETWORK=default

From e19aaadd425709b2673689565f4bb454c863b1a3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 24 Sep 2019 16:11:15 +0200
Subject: [PATCH 03/22] We probably always want SSD because we can start small,
 with 1GB

---
 gke/storageclasses/default.yaml                    |  6 ++++++
 gke/storageclasses/kustomization.yaml              |  6 ++++++
 gke/storageclasses/standard-non-default.yaml       | 10 ++++++++++
 gke/storageclasses/zonal-ssd-resizable-delete.yaml |  9 +++++++++
 gke/storageclasses/zonal-ssd-resizable-retain.yaml |  9 +++++++++
 5 files changed, 40 insertions(+)
 create mode 100644 gke/storageclasses/default.yaml
 create mode 100644 gke/storageclasses/kustomization.yaml
 create mode 100644 gke/storageclasses/standard-non-default.yaml
 create mode 100644 gke/storageclasses/zonal-ssd-resizable-delete.yaml
 create mode 100644 gke/storageclasses/zonal-ssd-resizable-retain.yaml

diff --git a/gke/storageclasses/default.yaml b/gke/storageclasses/default.yaml
new file mode 100644
index 00000000..92bd9a93
--- /dev/null
+++ b/gke/storageclasses/default.yaml
@@ -0,0 +1,6 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: zonal-ssd-resizable-delete
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
diff --git a/gke/storageclasses/kustomization.yaml b/gke/storageclasses/kustomization.yaml
new file mode 100644
index 00000000..1143f2da
--- /dev/null
+++ b/gke/storageclasses/kustomization.yaml
@@ -0,0 +1,6 @@
+resources:
+- standard-non-default.yaml
+- zonal-ssd-resizable-retain.yaml
+- zonal-ssd-resizable-delete.yaml
+patchesStrategicMerge:
+- default.yaml
diff --git a/gke/storageclasses/standard-non-default.yaml b/gke/storageclasses/standard-non-default.yaml
new file mode 100644
index 00000000..33662f5b
--- /dev/null
+++ b/gke/storageclasses/standard-non-default.yaml
@@ -0,0 +1,10 @@
+# based on a fresh GKE cluster's kubectl get storageclass standard -o yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: standard
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "false"
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-standard
diff --git a/gke/storageclasses/zonal-ssd-resizable-delete.yaml b/gke/storageclasses/zonal-ssd-resizable-delete.yaml
new file mode 100644
index 00000000..40bfc221
--- /dev/null
+++ b/gke/storageclasses/zonal-ssd-resizable-delete.yaml
@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: zonal-ssd-resizable-delete
+provisioner: kubernetes.io/gce-pd
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+parameters:
+  type: pd-ssd
diff --git a/gke/storageclasses/zonal-ssd-resizable-retain.yaml b/gke/storageclasses/zonal-ssd-resizable-retain.yaml
new file mode 100644
index 00000000..7ea1a399
--- /dev/null
+++ b/gke/storageclasses/zonal-ssd-resizable-retain.yaml
@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: zonal-ssd-resizable-retain
+provisioner: kubernetes.io/gce-pd
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+parameters:
+  type: pd-ssd

From 3600e3a3ddaf0503d3fc6c950998e96604b78f59 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 24 Sep 2019 16:16:31 +0200
Subject: [PATCH 04/22] Use defaults for stuff that y-stack isn't opinionated
 about

---
 bin/y-cluster-provision-gke | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 59f1edf7..e5a456ad 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -4,6 +4,7 @@ set -e
 
 NAME="$1"
 [ -z "$NAME" ] && echo "First argument must be a cluster name" && exit 1
+shift 1
 
 FLAGS=""
 
@@ -18,9 +19,6 @@ gcloud container get-server-config --region=$REGION | grep "1.14"
 
 FLAGS="$FLAGS --cluster-version=1.14.6-gke.1"
 FLAGS="$FLAGS --image-type=UBUNTU_CONTAINERD"
-FLAGS="$FLAGS --machine-type=g1-small"
-gcloud beta compute zones describe $REGION-a --format="value(availableCpuPlatforms)"
-#FLAGS="$FLAGS --min-cpu-platform 'Intel Skylake'"
 
 FLAGS="$FLAGS --num-nodes=1"
 FLAGS="$FLAGS --enable-autoscaling"
@@ -84,7 +82,7 @@ echo "### What this script will try to do is ..."
 echo "Use the currently configured project: $PROJECT"
 echo "Use the currently configured region: $REGION"
 echo "Add authentication to KUBECONFIG=$KUBECONFIG"
-echo "Create a GKE cluster using the following flags: $FLAGS"
+echo "Create a GKE cluster using the following flags: $FLAGS $@"
 echo ""
 
 read -p 'Please confirm [y/n]? ' yn
@@ -94,10 +92,7 @@ case $yn in
 esac
 
 #gcloud container clusters create "$NAME" $FLAGS
-gcloud container clusters create "$NAME" \
-  --min-cpu-platform 'Intel Skylake' \
-  --create-subnetwork "" \
-  $FLAGS
+gcloud container clusters create "$NAME" $FLAGS $@
 
 GKE_CONTEXT_NAME=$(kubectl config current-context)
 

From 8ce5b1f36185641fd0ce0c1d2ca5d5d08fa57476 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 24 Sep 2019 20:59:32 +0200
Subject: [PATCH 05/22] Struggling with values that are empty string or
 contains spaces

---
 bin/y-cluster-provision-gke | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index e5a456ad..2dc57533 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -91,8 +91,7 @@ case $yn in
   * ) exit;;
 esac
 
-#gcloud container clusters create "$NAME" $FLAGS
-gcloud container clusters create "$NAME" $FLAGS $@
+gcloud container clusters create "$NAME" $FLAGS "$@"
 
 GKE_CONTEXT_NAME=$(kubectl config current-context)
 

From d6680a7d8012474a91a65c40a8a3a9fb3baec816 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 07:33:00 +0200
Subject: [PATCH 06/22] Deal with globally unique bucket names

---
 bin/y-cluster-provision-gke                 | 25 +++++++++++++++++----
 gke/registry/.gitignore                     |  1 +
 gke/registry/builds-gcs-bucket-storage.yaml | 15 -------------
 3 files changed, 22 insertions(+), 19 deletions(-)
 delete mode 100644 gke/registry/builds-gcs-bucket-storage.yaml

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 2dc57533..b0476030 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -114,6 +114,8 @@ echo gcloud compute routers delete nat-router --region $REGION
 
 ### To be moved to registry provision script, but we do want the initial context name
 
+[ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
+
 gcloud compute backend-buckets list
 gsutil ls
 # https://cloud.google.com/config-connector/docs/overview
@@ -123,11 +125,26 @@ cat << EOF > gke/registry/builds-gcs-storagebucket.yaml
 apiVersion: storage.cnrm.cloud.google.com/v1alpha2
 kind: StorageBucket
 metadata:
-  name: ${GKE_CONTEXT_NAME}_builds
+  name: ${BUILDS_BUCKET_NAME}
 spec: {}
 EOF
 
-# TODO we also need to generate a bucket name, or a secret with a bucket name, for gke/registry/builds-gcs-bucket-storage.yaml
-
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
-  gsutil mb -c regional -l $REGION gs://${GKE_CONTEXT_NAME}_builds
+  gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME}
+
+cat << EOF > gke/registry/builds-gcs-storagebucket.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: registry
+spec:
+  template:
+    spec:
+      containers:
+      - name: docker-v2
+        env:
+        - name: REGISTRY_STORAGE
+          value: gcs
+        - name: REGISTRY_STORAGE_GCS_BUCKET
+          value: ${BUILDS_BUCKET_NAME}
+EOF
diff --git a/gke/registry/.gitignore b/gke/registry/.gitignore
index f39d4efd..3f20aa41 100644
--- a/gke/registry/.gitignore
+++ b/gke/registry/.gitignore
@@ -1,2 +1,3 @@
 # generated
 builds-gcs-storagebucket.yaml
+builds-gcs-bucket-storage.yaml
diff --git a/gke/registry/builds-gcs-bucket-storage.yaml b/gke/registry/builds-gcs-bucket-storage.yaml
deleted file mode 100644
index 5d02355c..00000000
--- a/gke/registry/builds-gcs-bucket-storage.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: registry
-spec:
-  template:
-    spec:
-      containers:
-      - name: docker-v2
-        image: registry:2.7.1@sha256:a54bc9be148764891c44676ce8c44f1e53514c43b1bfbab87b896f4b9f0b5d99
-        env:
-        - name: REGISTRY_STORAGE
-          value: gcs
-        - name: REGISTRY_STORAGE_GCS_BUCKET
-          value: # TODO must be generated by the provision script

From 6bc5b3dc49dd9959d5222dd6e26cb8406290fd0d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 08:13:51 +0200
Subject: [PATCH 07/22] Experimental containerd restart

---
 registry/node-update-containerd/daemonset.yaml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index f2bcf551..2b385388 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -11,6 +11,8 @@ spec:
       labels:
         app: containerd-registry-http
     spec:
+      hostPID: true
+      hostNetwork: true
       initContainers:
       - name: update
         image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
@@ -23,14 +25,24 @@ spec:
         #to troubleshoot use: - -cex
         - -ce
         - |
+          tail -f /dev/null
           cat << EOF >> /etc/containerd/config.toml
           [plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]
              endpoint = ["http://builds-registry.ystack.svc.cluster.local"]
           [plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]
              endpoint = ["http://prod-registry.ystack.svc.cluster.local"]
           EOF
-          echo "TODO currently you must manually ssh to the node and run: systemctl restart containerd"
-          # maybe https://alexei-led.github.io/post/k8s_node_shell/ ?
+      - name: containerd-restart
+        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
+        securityContext:
+          privileged: true
+        command:
+        - nsenter
+        - --mount=/proc/1/ns/mnt
+        - --
+        - systemctl
+        - restart
+        - containerd
       containers:
       - name: init-container-did-the-work
         image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610

From 5087f8180c590939591fbf730b09fc235df81721 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 08:22:52 +0200
Subject: [PATCH 08/22] Minio is optional but hosts update would crashloop
 without the service

---
 registry/node-update-hosts/daemonset.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/registry/node-update-hosts/daemonset.yaml b/registry/node-update-hosts/daemonset.yaml
index 56404a13..6ad2e1e6 100644
--- a/registry/node-update-hosts/daemonset.yaml
+++ b/registry/node-update-hosts/daemonset.yaml
@@ -23,7 +23,6 @@ spec:
           value: >-
             builds-registry.ystack.svc.cluster.local
             prod-registry.ystack.svc.cluster.local
-            blobs-minio.ystack.svc.cluster.local
         command:
         - bash
         - -ce

From d24ebb16a86c88c9cb008fc04ae092d428eea7f5 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 08:23:54 +0200
Subject: [PATCH 09/22] Adapts converge-generic to GKE

---
 converge-gke/kustomization.yaml | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 converge-gke/kustomization.yaml

diff --git a/converge-gke/kustomization.yaml b/converge-gke/kustomization.yaml
new file mode 100644
index 00000000..a4e23e06
--- /dev/null
+++ b/converge-gke/kustomization.yaml
@@ -0,0 +1,9 @@
+namespace: ystack
+bases:
+- ../buildkit
+- ../gke/registry
+- ../registry/node-update-hosts
+# Two ways to get nodes' containerd to allow pull from in-cluster .local service
+# (assuming we already have DNS name lookup fixed through for example ../registry/hosts-update):
+#- ../../registry/generic,tls/       # requires manual containerd restart until https://github.com/containerd/containerd/issues/3071 is resolved
+- ../registry/node-update-containerd # also requires restart but needs no TLS support in registry so might be a simpler option until we have containerd#3071

From 06747487ff66120fb10fed3cc58f27b66519f908 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 11:28:54 +0200
Subject: [PATCH 10/22] Using a secret would probably be more appropriatet

---
 bin/y-cluster-provision-gke | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index b0476030..151a2996 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -119,7 +119,6 @@ echo gcloud compute routers delete nat-router --region $REGION
 gcloud compute backend-buckets list
 gsutil ls
 # https://cloud.google.com/config-connector/docs/overview
-mkdir -p gke/registry
 cat << EOF > gke/registry/builds-gcs-storagebucket.yaml
 # https://github.com/GoogleCloudPlatform/k8s-config-connector/blob/master/resources/storagebucket/storage_v1alpha2_storagebucket.yaml
 apiVersion: storage.cnrm.cloud.google.com/v1alpha2
@@ -132,7 +131,7 @@ EOF
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME}
 
-cat << EOF > gke/registry/builds-gcs-storagebucket.yaml
+cat << EOF > gke/registry/builds-gcs-bucket-storage.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:

From 60828002df574277e158595147cb844f6d4328e6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 11:29:15 +0200
Subject: [PATCH 11/22] Append to containerd config only once

---
 .../node-update-containerd/daemonset.yaml     | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index 2b385388..ffee926e 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -20,18 +20,23 @@ spec:
         - name: etc-containerd
           mountPath: /etc/containerd
           readOnly: false
+        env:
+        - name: CONFIG_PATH
+          value: /etc/containerd/config.toml
         command:
         - bash
         #to troubleshoot use: - -cex
         - -ce
         - |
-          tail -f /dev/null
-          cat << EOF >> /etc/containerd/config.toml
-          [plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]
-             endpoint = ["http://builds-registry.ystack.svc.cluster.local"]
-          [plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]
-             endpoint = ["http://prod-registry.ystack.svc.cluster.local"]
-          EOF
+          if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then
+            echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH
+            echo '  endpoint = ["http://builds-registry.ystack.svc.cluster.local"'           >> $CONFIG_PATH
+          fi
+          if ! grep prod-registry.ystack.svc.cluster.local $CONFIG_PATH; then
+            echo '[plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]'   >> $CONFIG_PATH
+            echo '  endpoint = ["http://prod-registry.ystack.svc.cluster.local"'             >> $CONFIG_PATH
+          fi
+          cat $CONFIG_PATH
       - name: containerd-restart
         image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
         securityContext:

From d8a640a14e6864c0de3365d10e5a24f1bd8bbb73 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 11:29:46 +0200
Subject: [PATCH 12/22] This decision has moved to converge-gke

---
 gke/registry/kustomization.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/gke/registry/kustomization.yaml b/gke/registry/kustomization.yaml
index 034350cb..ddf39ee5 100644
--- a/gke/registry/kustomization.yaml
+++ b/gke/registry/kustomization.yaml
@@ -1,8 +1,4 @@
 bases:
 - ../../registry/generic/
-# Two ways to get nodes' containerd to allow pull from in-cluster .local service
-# (assuming we already have DNS name lookup fixed through for example ../registry/hosts-update):
-#- ../../registry/generic,tls/       # requires manual containerd restart until https://github.com/containerd/containerd/issues/3071 is resolved
-- ../../registry/node-update-containerd/ # also requires restart but needs no TLS support in registry so might be a simpler option until we have containerd#3071
 patchesStrategicMerge:
 - builds-gcs-bucket-storage.yaml

From e53f6d73a51a169919c9a366267057d6b5750e5c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 20:37:15 +0200
Subject: [PATCH 13/22] Fix file paths for provision called from somewhere else

---
 bin/y-cluster-provision-gke | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 151a2996..df9d4038 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -119,7 +119,7 @@ echo gcloud compute routers delete nat-router --region $REGION
 gcloud compute backend-buckets list
 gsutil ls
 # https://cloud.google.com/config-connector/docs/overview
-cat << EOF > gke/registry/builds-gcs-storagebucket.yaml
+cat << EOF > $YSTACK_HOME/gke/registry/builds-gcs-storagebucket.yaml
 # https://github.com/GoogleCloudPlatform/k8s-config-connector/blob/master/resources/storagebucket/storage_v1alpha2_storagebucket.yaml
 apiVersion: storage.cnrm.cloud.google.com/v1alpha2
 kind: StorageBucket
@@ -131,7 +131,7 @@ EOF
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME}
 
-cat << EOF > gke/registry/builds-gcs-bucket-storage.yaml
+cat << EOF > $YSTACK_HOME/gke/registry/builds-gcs-bucket-storage.yaml
 apiVersion: apps/v1
 kind: Deployment
 metadata:

From 5cf22aeddd41800342259a8d87645be503db3fc0 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 21:05:57 +0200
Subject: [PATCH 14/22] Use a secret as referrable bucket name

---
 bin/y-cluster-provision-gke                 | 17 +----------------
 gke/registry/.gitignore                     |  1 -
 gke/registry/builds-gcs-bucket-storage.yaml | 17 +++++++++++++++++
 3 files changed, 18 insertions(+), 17 deletions(-)
 create mode 100644 gke/registry/builds-gcs-bucket-storage.yaml

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index df9d4038..994898be 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -131,19 +131,4 @@ EOF
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME}
 
-cat << EOF > $YSTACK_HOME/gke/registry/builds-gcs-bucket-storage.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: registry
-spec:
-  template:
-    spec:
-      containers:
-      - name: docker-v2
-        env:
-        - name: REGISTRY_STORAGE
-          value: gcs
-        - name: REGISTRY_STORAGE_GCS_BUCKET
-          value: ${BUILDS_BUCKET_NAME}
-EOF
+kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
diff --git a/gke/registry/.gitignore b/gke/registry/.gitignore
index 3f20aa41..f39d4efd 100644
--- a/gke/registry/.gitignore
+++ b/gke/registry/.gitignore
@@ -1,3 +1,2 @@
 # generated
 builds-gcs-storagebucket.yaml
-builds-gcs-bucket-storage.yaml
diff --git a/gke/registry/builds-gcs-bucket-storage.yaml b/gke/registry/builds-gcs-bucket-storage.yaml
new file mode 100644
index 00000000..1badf5a2
--- /dev/null
+++ b/gke/registry/builds-gcs-bucket-storage.yaml
@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: registry
+spec:
+  template:
+    spec:
+      containers:
+      - name: docker-v2
+        env:
+        - name: REGISTRY_STORAGE
+          value: gcs
+        - name: REGISTRY_STORAGE_GCS_BUCKET
+          valueFrom:
+            secretKeyRef:
+              name: registry-persistence
+              key: gcsBucketName

From 62b4e89218c9faa5a790c09b5f230d7af6b2771e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 21:08:25 +0200
Subject: [PATCH 15/22] Restart containerd only if config has changed

because I think these restarts might cause instability,
in particular if there's other containers on the node.
The purpose is to update+restart immediately after node start.
---
 .../node-update-containerd/daemonset.yaml     | 23 ++++++++++---------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index ffee926e..872e1df4 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -16,6 +16,8 @@ spec:
       initContainers:
       - name: update
         image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
+        securityContext:
+          privileged: true
         volumeMounts:
         - name: etc-containerd
           mountPath: /etc/containerd
@@ -31,23 +33,22 @@ spec:
           if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then
             echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH
             echo '  endpoint = ["http://builds-registry.ystack.svc.cluster.local"'           >> $CONFIG_PATH
+            touch /results/config-updated
           fi
           if ! grep prod-registry.ystack.svc.cluster.local $CONFIG_PATH; then
             echo '[plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]'   >> $CONFIG_PATH
             echo '  endpoint = ["http://prod-registry.ystack.svc.cluster.local"'             >> $CONFIG_PATH
+            touch /results/config-updated
           fi
           cat $CONFIG_PATH
-      - name: containerd-restart
-        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
-        securityContext:
-          privileged: true
-        command:
-        - nsenter
-        - --mount=/proc/1/ns/mnt
-        - --
-        - systemctl
-        - restart
-        - containerd
+          if [ -f /results/config-updated ]; then
+            echo "Configuration has been updated. Restarting containerd (not expecting any output) ..."
+            nsenter --mount=/proc/1/ns/mnt -- systemctl restart containerd
+            echo "Restart triggered."
+          else
+            echo "No configration change"
+          fi
+          echo "Done"
       containers:
       - name: init-container-did-the-work
         image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610

From 29e74f417d0c31844797b251ded8dca8a44b38ce Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 25 Sep 2019 21:35:42 +0200
Subject: [PATCH 16/22] Still not predictable

---
 bin/y-cluster-provision-gke                    | 4 ++++
 registry/node-update-containerd/daemonset.yaml | 6 +++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 994898be..54569489 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -112,6 +112,10 @@ echo gcloud container clusters delete --zone $REGION-a $NAME
 echo gcloud compute routers nats delete --router-region $REGION --router nat-router nat-config
 echo gcloud compute routers delete nat-router --region $REGION
 
+kubectl create namespace ystack
+
+kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
+
 ### To be moved to registry provision script, but we do want the initial context name
 
 [ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index 872e1df4..aa0ad81f 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -22,13 +22,15 @@ spec:
         - name: etc-containerd
           mountPath: /etc/containerd
           readOnly: false
+        - name: results
+          mountPath: /results
         env:
         - name: CONFIG_PATH
           value: /etc/containerd/config.toml
         command:
         - bash
         #to troubleshoot use: - -cex
-        - -ce
+        - -cex
         - |
           if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then
             echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH
@@ -57,3 +59,5 @@ spec:
       - name: etc-containerd
         hostPath:
           path: /etc/containerd
+      - name: results
+        emptyDir: {}

From 723ddb7c8821449ad191d3736108ed39564d8cbb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Sep 2019 07:06:49 +0200
Subject: [PATCH 17/22] Reverted to restart in separate container, but I must
 have been

very lucky that time that it worked because now I always get unready nodes
---
 .../node-update-containerd/daemonset.yaml     | 38 +++++++++++++++----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index aa0ad81f..e47ff80a 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -16,8 +16,6 @@ spec:
       initContainers:
       - name: update
         image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
-        securityContext:
-          privileged: true
         volumeMounts:
         - name: etc-containerd
           mountPath: /etc/containerd
@@ -43,14 +41,40 @@ spec:
             touch /results/config-updated
           fi
           cat $CONFIG_PATH
+      - name: pause-if-no-config-changes
+        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
+        volumeMounts:
+        - name: results
+          mountPath: /results
+        command:
+        - bash
+        - -ce
+        - |
           if [ -f /results/config-updated ]; then
-            echo "Configuration has been updated. Restarting containerd (not expecting any output) ..."
-            nsenter --mount=/proc/1/ns/mnt -- systemctl restart containerd
-            echo "Restart triggered."
+            echo "Configuration seems to have changed. Let's proceed with restart."
           else
-            echo "No configration change"
+            echo "No configration change. Let's not restart as it could affect stability. Sleeping forever ..."
+            sleep infinity
           fi
-          echo "Done"
+      - name: containerd-restart
+        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
+        securityContext:
+          privileged: true
+        command:
+        - nsenter
+        - --mount=/proc/1/ns/mnt
+        - --
+        - systemctl
+        - restart
+        - containerd
+      - name: pause-always
+        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
+        volumeMounts:
+        - name: results
+          mountPath: /results
+        command:
+        - sleep
+        - infinity
       containers:
       - name: init-container-did-the-work
         image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610

From 5fe3ad185683cc5959f4e5b04b92835fae2f61e6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Sep 2019 07:25:01 +0200
Subject: [PATCH 18/22] Crap. We don't validate toml syntax and systemctl
 restart is mute.

---
 registry/node-update-containerd/daemonset.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index e47ff80a..f12af5d9 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -32,12 +32,12 @@ spec:
         - |
           if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then
             echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH
-            echo '  endpoint = ["http://builds-registry.ystack.svc.cluster.local"'           >> $CONFIG_PATH
+            echo '  endpoint = ["http://builds-registry.ystack.svc.cluster.local"]'          >> $CONFIG_PATH
             touch /results/config-updated
           fi
           if ! grep prod-registry.ystack.svc.cluster.local $CONFIG_PATH; then
             echo '[plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]'   >> $CONFIG_PATH
-            echo '  endpoint = ["http://prod-registry.ystack.svc.cluster.local"'             >> $CONFIG_PATH
+            echo '  endpoint = ["http://prod-registry.ystack.svc.cluster.local"]'            >> $CONFIG_PATH
             touch /results/config-updated
           fi
           cat $CONFIG_PATH

From d2fa48965d57bcfdc9a9e67d2c724337f302d0e6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Sep 2019 07:30:33 +0200
Subject: [PATCH 19/22] With a valid config file this worked already

---
 .../node-update-containerd/daemonset.yaml     | 46 +++++--------------
 1 file changed, 11 insertions(+), 35 deletions(-)

diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml
index f12af5d9..294ac914 100644
--- a/registry/node-update-containerd/daemonset.yaml
+++ b/registry/node-update-containerd/daemonset.yaml
@@ -25,10 +25,12 @@ spec:
         env:
         - name: CONFIG_PATH
           value: /etc/containerd/config.toml
+        securityContext:
+          privileged: true
         command:
         - bash
         #to troubleshoot use: - -cex
-        - -cex
+        - -ce
         - |
           if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then
             echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH
@@ -41,40 +43,14 @@ spec:
             touch /results/config-updated
           fi
           cat $CONFIG_PATH
-      - name: pause-if-no-config-changes
-        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
-        volumeMounts:
-        - name: results
-          mountPath: /results
-        command:
-        - bash
-        - -ce
-        - |
-          if [ -f /results/config-updated ]; then
-            echo "Configuration seems to have changed. Let's proceed with restart."
-          else
-            echo "No configration change. Let's not restart as it could affect stability. Sleeping forever ..."
-            sleep infinity
-          fi
-      - name: containerd-restart
-        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
-        securityContext:
-          privileged: true
-        command:
-        - nsenter
-        - --mount=/proc/1/ns/mnt
-        - --
-        - systemctl
-        - restart
-        - containerd
-      - name: pause-always
-        image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58
-        volumeMounts:
-        - name: results
-          mountPath: /results
-        command:
-        - sleep
-        - infinity
+           if [ -f /results/config-updated ]; then
+            echo "Configuration has been updated; restarting containerd (not expecting any output) ..."
+            nsenter --mount=/proc/1/ns/mnt -- systemctl restart containerd
+            echo "Restart triggered"
+           else
+            echo "No configration change"
+           fi
+          echo "Done"
       containers:
       - name: init-container-did-the-work
         image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610

From 0725c53c84d691b4ffaa84f26a43fce8dea264d1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Sep 2019 07:36:42 +0200
Subject: [PATCH 20/22] Create the secret first, in case the bucket already
 exists

---
 bin/y-cluster-provision-gke | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 54569489..22426c42 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -120,6 +120,8 @@ kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
 
 [ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
 
+kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
+
 gcloud compute backend-buckets list
 gsutil ls
 # https://cloud.google.com/config-connector/docs/overview
@@ -134,5 +136,3 @@ EOF
 
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME}
-
-kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}

From 02fe8c73ac2a67e80abc8fd47a8bbdef89acf295 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Sep 2019 07:36:55 +0200
Subject: [PATCH 21/22] We always peg our images

---
 examples/basic-dev-inner-loop/backend/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/basic-dev-inner-loop/backend/Dockerfile b/examples/basic-dev-inner-loop/backend/Dockerfile
index 63001ecb..19f539db 100644
--- a/examples/basic-dev-inner-loop/backend/Dockerfile
+++ b/examples/basic-dev-inner-loop/backend/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:10.15.3-alpine
+FROM node:10.16.3-alpine@sha256:77c898d0da5e7bfb6e05c9a64de136ba4e03889a72f3c298e95df822a38f450d
 
 WORKDIR /app
 EXPOSE 3000

From db0d39587509b63c57e658f953d53f537bb1f8c6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Sep 2019 08:04:00 +0200
Subject: [PATCH 22/22] Good enough now for
 source-to-url-by-build-context-transfer

---
 bin/y-cluster-provision-gke     | 3 +--
 converge-gke/kustomization.yaml | 1 +
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 22426c42..c6f86a37 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -107,8 +107,7 @@ echo gcloud compute routers nats create nat-config \
     --auto-allocate-nat-external-ips
 
 echo "To clean up:"
-#echo gcloud container clusters delete --region $REGION $NAME
-echo gcloud container clusters delete --zone $REGION-a $NAME
+echo gcloud container clusters delete --zone $REGION $NAME
 echo gcloud compute routers nats delete --router-region $REGION --router nat-router nat-config
 echo gcloud compute routers delete nat-router --region $REGION
 
diff --git a/converge-gke/kustomization.yaml b/converge-gke/kustomization.yaml
index a4e23e06..67820f0d 100644
--- a/converge-gke/kustomization.yaml
+++ b/converge-gke/kustomization.yaml
@@ -1,5 +1,6 @@
 namespace: ystack
 bases:
+- ../gke/storageclasses
 - ../buildkit
 - ../gke/registry
 - ../registry/node-update-hosts