From ed4687c9f725132b45e6e6003989167e6ac4a9e4 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 1 Nov 2019 09:14:16 +0100
Subject: [PATCH 01/39] Currently I'd bet on k3d for y-stack CI

---
 bin/y-k3d | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100755 bin/y-k3d

diff --git a/bin/y-k3d b/bin/y-k3d
new file mode 100755
index 00000000..c4a52a8f
--- /dev/null
+++ b/bin/y-k3d
@@ -0,0 +1,17 @@
+#!/bin/sh
+[ -z "$DEBUG" ] || set -x
+set -e
+
+version=1.3.4
+
+bin_name=k3d \
+  bin_version=v${version} \
+  Darwin_url=https://github.com/rancher/k3d/releases/download/v${version}/k3d-darwin-amd64 \
+  Darwin_sha256=011072c5be4f33af606873aa7e326246f2ee1a158c07ed4f2791eb2e3de184e3 \
+  Linux_url=https://github.com/rancher/k3d/releases/download/v${version}/k3d-linux-amd64 \
+  Linux_sha256=41e0579b1499f6051e9d0d8a6d39d5f419e1037851c0121b676190af5825f4c0 \
+  y-bin-dependency-download || exit $?
+
+echo "NOTE y-stack on k3s is quite blocked by https://github.com/rancher/k3d/issues/134"
+
+y-k3d-v${version}-bin "$@" || echo $?

From 6a3647ccff70ffcd4b6a854b387d101fd48599ac Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 2 Nov 2019 04:14:01 +0100
Subject: [PATCH 02/39] Current
 github.com/rancher/k3s/blob/master/docker-compose.yml

---
 docker-compose.test.yml | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 docker-compose.test.yml

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
new file mode 100644
index 00000000..7d83ddc5
--- /dev/null
+++ b/docker-compose.test.yml
@@ -0,0 +1,30 @@
+version: '3'
+services:
+  server:
+    image: rancher/k3s:v0.10.0
+    command: server --disable-agent
+    environment:
+    - K3S_CLUSTER_SECRET=somethingtotallyrandom
+    - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
+    - K3S_KUBECONFIG_MODE=666
+    volumes:
+    - k3s-server:/var/lib/rancher/k3s
+    # This is just so that we get the kubeconfig file out
+    - .:/output
+    ports:
+    - 6443:6443
+
+  node:
+    image: rancher/k3s:v0.10.0
+    tmpfs:
+    - /run
+    - /var/run
+    privileged: true
+    environment:
+    - K3S_URL=https://server:6443
+    - K3S_CLUSTER_SECRET=somethingtotallyrandom
+    # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
+    #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
+
+volumes:
+  k3s-server: {}

From 441338e13a95986533700f58a83d7bac6cd904f8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 2 Nov 2019 04:20:02 +0100
Subject: [PATCH 03/39] k3s v0.10.2 official image

---
 docker-compose.test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 7d83ddc5..0f656e32 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -1,7 +1,7 @@
 version: '3'
 services:
   server:
-    image: rancher/k3s:v0.10.0
+    image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
     command: server --disable-agent
     environment:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
@@ -15,7 +15,7 @@ services:
     - 6443:6443
 
   node:
-    image: rancher/k3s:v0.10.0
+    image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
     tmpfs:
     - /run
     - /var/run

From 50cfa0bad7841596ad0952f2248a81a54f3cfacf Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 2 Nov 2019 04:36:35 +0100
Subject: [PATCH 04/39] For
 docs.docker.com/docker-hub/builds/automated-testing/

---
 docker-compose.test.yml | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 0f656e32..d30069a8 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -5,14 +5,14 @@ services:
     command: server --disable-agent
     environment:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
-    - K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml
+    - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml
     - K3S_KUBECONFIG_MODE=666
     volumes:
     - k3s-server:/var/lib/rancher/k3s
-    # This is just so that we get the kubeconfig file out
-    - .:/output
-    ports:
-    - 6443:6443
+    - admin:/admin
+    # Host port not needed for CI
+    #ports:
+    #- 6443:6443
 
   node:
     image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
@@ -26,5 +26,23 @@ services:
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
 
+  sut:
+    image: lachlanevenson/k8s-kubectl:v1.16.2@sha256:a0ec573235c11c6be806d2a8f730666e9627fd5c66984b3b0ea8c99d8c8952c0
+    environment:
+    - KUBECONFIG=/admin/.kube/config
+    volumes:
+    - admin:/admin
+    entrypoint:
+    - /bin/sh
+    - -cex
+    command:
+    - |
+      cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
+      # TODO must retry until server is up
+      kubectl get pods --all-namespaces
+      echo "Doing nothing ATM"
+      tail -f /dev/null
+
 volumes:
   k3s-server: {}
+  admin: {}

From 0d6ca5274d3d8d7e447263c84f005c36f266e87f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 2 Nov 2019 04:50:24 +0100
Subject: [PATCH 05/39] The cost of a node is ~5% CPU and 100 MiB memory
 (docker stats)

---
 docker-compose.test.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index d30069a8..f8a5a078 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -14,7 +14,8 @@ services:
     #ports:
     #- 6443:6443
 
-  node:
+  node1:
+    hostname: node1
     image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
     tmpfs:
     - /run
@@ -26,6 +27,17 @@ services:
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
 
+  node2:
+    hostname: node2
+    image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
+    tmpfs:
+    - /run
+    - /var/run
+    privileged: true
+    environment:
+    - K3S_URL=https://server:6443
+    - K3S_CLUSTER_SECRET=somethingtotallyrandom
+
   sut:
     image: lachlanevenson/k8s-kubectl:v1.16.2@sha256:a0ec573235c11c6be806d2a8f730666e9627fd5c66984b3b0ea8c99d8c8952c0
     environment:
@@ -39,7 +51,7 @@ services:
     - |
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
       # TODO must retry until server is up
-      kubectl get pods --all-namespaces
+      kubectl get nodes
       echo "Doing nothing ATM"
       tail -f /dev/null
 

From cf428b7acea739e86a595b0bcd6a458f07132c0e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 2 Nov 2019 21:21:03 +0100
Subject: [PATCH 06/39] Now we can add capacity in 2 GB memory increments

---
 docker-compose.test.yml | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index f8a5a078..b65b9591 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -14,8 +14,7 @@ services:
     #ports:
     #- 6443:6443
 
-  node1:
-    hostname: node1
+  node:
     image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
     tmpfs:
     - /run
@@ -27,17 +26,6 @@ services:
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
 
-  node2:
-    hostname: node2
-    image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
-    tmpfs:
-    - /run
-    - /var/run
-    privileged: true
-    environment:
-    - K3S_URL=https://server:6443
-    - K3S_CLUSTER_SECRET=somethingtotallyrandom
-
   sut:
     image: lachlanevenson/k8s-kubectl:v1.16.2@sha256:a0ec573235c11c6be806d2a8f730666e9627fd5c66984b3b0ea8c99d8c8952c0
     environment:

From 544268410dc7252aa62444aa4b2c49c76319f33a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 2 Nov 2019 21:23:57 +0100
Subject: [PATCH 07/39] Scale to X: NODES=X y-cluster-provision-k3s-docker

---
 bin/y-cluster-provision-k3s-docker | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100755 bin/y-cluster-provision-k3s-docker

diff --git a/bin/y-cluster-provision-k3s-docker b/bin/y-cluster-provision-k3s-docker
new file mode 100755
index 00000000..5bf7293f
--- /dev/null
+++ b/bin/y-cluster-provision-k3s-docker
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+[ -z "$KUBECONFIG" ] && echo "Provision requires an explicit KUBECONFIG env" && exit 1
+
+COMPOSE="docker-compose -f docker-compose.test.yml -p ystack"
+[ -z "$NODES" ] && NODES=2
+
+(cd $YSTACK_HOME/ && \
+  $COMPOSE up --scale node=$NODES -d && \
+  docker ps && \
+  echo "# Compose commmand:" && echo "$COMPOSE" && \
+  echo "# To tear down:" && echo "$COMPOSE down --remove-orphans -v" && \
+  echo "# Currently, because sut doesn't wait for server to be up, you need another:" && echo "$COMPOSE up -d" && \
+  true )

From b83af88e19c438dc0fad501e3503142e80effce8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 3 Nov 2019 05:09:15 +0100
Subject: [PATCH 08/39] Unrelated dockerd upgrade

---
 docker/dockerd-statefulset.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/dockerd-statefulset.yaml b/docker/dockerd-statefulset.yaml
index d75c1e18..a2185ef3 100644
--- a/docker/dockerd-statefulset.yaml
+++ b/docker/dockerd-statefulset.yaml
@@ -32,7 +32,7 @@ spec:
           ln -s /var/lib/docker/overlay2/var-lib-docker-image /var/lib/docker/image
       containers:
       - name: dockerd
-        image: docker:19.03.3-dind@sha256:5d11c2a1300b7438e21cbb5b2bd704e663b70534abe90c9072081dfb18e144de
+        image: docker:19.03.4-dind@sha256:156e4e723c80223298259dad9a925a71f385456b731c139bec12d29b3a74170c
         securityContext:
           privileged: true
         args:

From b8e3f3a81442dd89e38e5448d8271edaf2f53407 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 3 Nov 2019 05:09:53 +0100
Subject: [PATCH 09/39] Runner refresh

---
 runner.Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runner.Dockerfile b/runner.Dockerfile
index d486be39..71d6a043 100644
--- a/runner.Dockerfile
+++ b/runner.Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:19.10@sha256:a21f154506cc00974f647e13dbba6b7035da35c7669a4bb919515d895221face
+FROM ubuntu:19.10@sha256:a5193c15d7705bc2be91781355c4932321c06c18914facdd113d5bfcace7f92d
 
 RUN set -ex; \
   export DEBIAN_FRONTEND=noninteractive; \
@@ -12,10 +12,10 @@ RUN set -ex; \
 
 RUN set -ex; \
   F=$(mktemp); \
-  curl -SLs https://dl.k8s.io/v1.16.1/kubernetes-client-linux-amd64.tar.gz \
+  curl -SLs https://dl.k8s.io/v1.16.2/kubernetes-client-linux-amd64.tar.gz \
     | tee $F \
     | tar xzf - --strip-components=3 -C /usr/local/bin/; \
-  echo "e355a74a17d96785b0b217673e67fa0f02daa1939f10d410602ac0a0d061a4db71d727b67f75aa886007dab95dd5c7f8cc38253d291dc4d2504ce673df69fb32 $F" \
+  echo "69bb92c9b16c0286d7401d87cc73b85c88d6f9a17d2cf1748060e44525194b5be860daf7554c4c6319a546c9ff10f2b4df42a27e32d8f95a4052993b17ef57c0 $F" \
     | sha512sum -c -; \
   rm $F
 

From efe0fc27990a4998002e43377632e28a07a91fc6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 3 Nov 2019 05:42:51 +0100
Subject: [PATCH 10/39] Adds the indispensable :) kubectl "waitretry" plugin,

which probably anyone who knows more about backoffs or bash
than I do wouldn't write this way.
---
 bin/kubectl-waitretry          | 15 +++++++++++++++
 runner.Dockerfile.dockerignore |  1 +
 2 files changed, 16 insertions(+)
 create mode 100755 bin/kubectl-waitretry

diff --git a/bin/kubectl-waitretry b/bin/kubectl-waitretry
new file mode 100755
index 00000000..9426c790
--- /dev/null
+++ b/bin/kubectl-waitretry
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+[ -z "$WAITRETRIES" ] && WAITRETRIES=10
+
+SLEEP=1
+RETRY=0
+until kubectl wait $@; do
+  [ $RETRY -lt $WAITRETRIES ] || exit 1
+  echo "waitretry after $SLEEP seconds ..."
+  sleep $SLEEP
+  SLEEP=$(( $SLEEP * 2 ))
+  RETRY=$(( $RETRY + 1 ))
+done
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index 64a08b49..38973fd3 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -1,3 +1,4 @@
 *
 !bin/y-*
 bin/*-bin
+!bin/kubectl-

From 4f8b6d7499186fdd33945dc40792a974099f84cb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 3 Nov 2019 09:52:00 +0100
Subject: [PATCH 11/39] WIP converge using runner

---
 bin/y-cluster-provision-k3s-docker | 2 +-
 docker-compose.test.yml            | 9 ++++++---
 runner.Dockerfile.dockerignore     | 7 +++++++
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/bin/y-cluster-provision-k3s-docker b/bin/y-cluster-provision-k3s-docker
index 5bf7293f..5aedc67f 100755
--- a/bin/y-cluster-provision-k3s-docker
+++ b/bin/y-cluster-provision-k3s-docker
@@ -8,7 +8,7 @@ COMPOSE="docker-compose -f docker-compose.test.yml -p ystack"
 [ -z "$NODES" ] && NODES=2
 
 (cd $YSTACK_HOME/ && \
-  $COMPOSE up --scale node=$NODES -d && \
+  $COMPOSE up --scale node=$NODES -d --build && \
   docker ps && \
   echo "# Compose commmand:" && echo "$COMPOSE" && \
   echo "# To tear down:" && echo "$COMPOSE down --remove-orphans -v" && \
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index b65b9591..8df5123f 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -27,7 +27,10 @@ services:
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
 
   sut:
-    image: lachlanevenson/k8s-kubectl:v1.16.2@sha256:a0ec573235c11c6be806d2a8f730666e9627fd5c66984b3b0ea8c99d8c8952c0
+    build:
+      context: .
+      # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore
+      dockerfile: runner.Dockerfile
     environment:
     - KUBECONFIG=/admin/.kube/config
     volumes:
@@ -37,9 +40,9 @@ services:
     - -cex
     command:
     - |
+      until test -f /admin/.kube/kubeconfig.yaml; do echo "Waiting for a kubeconfig ..." && sleep 1; done
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
-      # TODO must retry until server is up
-      kubectl get nodes
+      kubectl-waitretry kubectl wait --for=condition=Ready node --all
       echo "Doing nothing ATM"
       tail -f /dev/null
 
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index 38973fd3..b5160c2b 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -2,3 +2,10 @@
 !bin/y-*
 bin/*-bin
 !bin/kubectl-
+
+# Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date:
+!converge-generic/
+!minio/
+!registry/generic/
+!registry/node-update-hosts/
+!registry/generic,minio/

From 1419df994d75375eaac3d516359a529b02ef9e28 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 4 Nov 2019 06:20:06 +0100
Subject: [PATCH 12/39] Current sorry state of dev mode

---
 bin/y-cluster-provision-k3s-docker | 22 +++++++++++++++++++++-
 docker-compose.dev-overrides.yml   | 11 +++++++++++
 docker-compose.test.yml            |  1 +
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 docker-compose.dev-overrides.yml

diff --git a/bin/y-cluster-provision-k3s-docker b/bin/y-cluster-provision-k3s-docker
index 5aedc67f..3680c46c 100755
--- a/bin/y-cluster-provision-k3s-docker
+++ b/bin/y-cluster-provision-k3s-docker
@@ -4,8 +4,15 @@ set -e
 
 [ -z "$KUBECONFIG" ] && echo "Provision requires an explicit KUBECONFIG env" && exit 1
 
-COMPOSE="docker-compose -f docker-compose.test.yml -p ystack"
+COMPOSE="docker-compose -p ystack -f docker-compose.test.yml"
 [ -z "$NODES" ] && NODES=2
+# TODO condition to toggle dev/qa mode
+false || {
+  COMPOSE="$COMPOSE -f docker-compose.dev-overrides.yml"
+  # ATM with port-forward we only support one node
+  [ $NODES -eq 1 ] || echo "# Dev mode only supports one node, util we add some kind of load balancer container"
+  NODES=1
+}
 
 (cd $YSTACK_HOME/ && \
   $COMPOSE up --scale node=$NODES -d --build && \
@@ -14,3 +21,16 @@ COMPOSE="docker-compose -f docker-compose.test.yml -p ystack"
   echo "# To tear down:" && echo "$COMPOSE down --remove-orphans -v" && \
   echo "# Currently, because sut doesn't wait for server to be up, you need another:" && echo "$COMPOSE up -d" && \
   true )
+
+cat << EOF
+# Misc notes
+
+docker-compose -p ystack -f docker-compose.test.yml -f docker-compose.dev-overrides.yml exec server sh -c 'cat /admin/.kube/kubeconfig.yaml'
+export KUBECONFIG=$(pwd)/k.yaml
+
+kubectl create namespace ystack
+kubectl apply -k converge-generic/
+
+# nodePort should match port forward
+kubectl -n kube-system edit service traefik
+EOF
diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
new file mode 100644
index 00000000..a25dcde4
--- /dev/null
+++ b/docker-compose.dev-overrides.yml
@@ -0,0 +1,11 @@
+version: '3'
+services:
+
+  server:
+    ports:
+    - 6443:6443
+
+  node:
+    ports:
+    - 80:30080
+    - 443:30443
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 8df5123f..473a438e 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -1,5 +1,6 @@
 version: '3'
 services:
+
   server:
     image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
     command: server --disable-agent

From bfa69f9b2edba798aa7f689b180b1ea9f8533057 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 14:00:14 +0100
Subject: [PATCH 13/39] Advancements by other parties

---
 bin/y-kubefwd                               | 8 +++++---
 bin/y-skaffold                              | 6 +++---
 examples/basic-dev-inner-loop/skaffold.yaml | 2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/bin/y-kubefwd b/bin/y-kubefwd
index c3f549b8..63783efa 100755
--- a/bin/y-kubefwd
+++ b/bin/y-kubefwd
@@ -3,12 +3,14 @@
 set -e
 YBIN="$(dirname $0)"
 
-version=1.9.2
+version=1.9.6
 
 bin_name=kubefwd \
   bin_version=v${version} \
-  Darwin_url=https://github.com/txn2/kubefwd/releases/download/v${version}/kubefwd_macOS_amd64.tar.gz \
-  Darwin_sha256=3f49349f7de2c8f60ae4d1248be4f936c9082d2b23cf9fa093caef20a60685d4 \
+  Darwin_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_Darwin_amd64.tar.gz \
+  Darwin_sha256=8a9a75d5377fbc8fc83152cdab4319b0e72f71ef084ef07703713b63e3ca3388 \
+  Linux_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_linux_amd64.tar.gz \
+  Linux_sha256=4a37c2f9763b46d2aba25cb6d07be36775213e208525911df27e398ea7d94d31 \
   bin_tgz_path=kubefwd \
   y-bin-dependency-download || exit $?
 
diff --git a/bin/y-skaffold b/bin/y-skaffold
index a30fb2c9..aa4b0dec 100755
--- a/bin/y-skaffold
+++ b/bin/y-skaffold
@@ -5,14 +5,14 @@ YBIN="$(dirname $0)"
 
 [ "$1" = "dev" ] && echo "y-skaffold dev requires -n [namespace] as initial args" && exit 1
 
-version=0.40.0
+version=1.0.0
 
 bin_name=skaffold \
   bin_version=v${version} \
   Darwin_url=https://github.com/GoogleContainerTools/skaffold/releases/download/v${version}/skaffold-darwin-amd64 \
-  Darwin_sha256=5175e11b36009cf5c1ba53f1564a2c1e66fd5a48a87e73a9bb672bade6c51174 \
+  Darwin_sha256=5a7b4dad8dfa83129c2076e799fa5a1a004f8c40cae0bb024828d5b542a96a84 \
   Linux_url=https://github.com/GoogleContainerTools/skaffold/releases/download/v${version}/skaffold-linux-amd64 \
-  Linux_sha256=f2d876e60cfc0dfb0fc79263bc9900e9c08a8300bf1e4b2bbfef3f9936cc78e0 \
+  Linux_sha256=f03cb2115cd510b6de8b024165b6621e453935faaf99b20c8cdf9933eec9b913 \
   y-bin-dependency-download || exit $?
 
 # https://skaffold.dev/docs/concepts/#local-development
diff --git a/examples/basic-dev-inner-loop/skaffold.yaml b/examples/basic-dev-inner-loop/skaffold.yaml
index 5d659be7..0e0a2c3b 100644
--- a/examples/basic-dev-inner-loop/skaffold.yaml
+++ b/examples/basic-dev-inner-loop/skaffold.yaml
@@ -1,4 +1,4 @@
-apiVersion: skaffold/v1beta16
+apiVersion: skaffold/v1
 kind: Config
 build:
   tagPolicy:

From 65d7f392ed375a2cb94226adf3b0493a6e6e1a55 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 15:13:48 +0100
Subject: [PATCH 14/39] Waitretry should have the same args as kubectl wait

---
 docker-compose.test.yml        | 4 ++--
 runner.Dockerfile              | 1 +
 runner.Dockerfile.dockerignore | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 473a438e..31a99113 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -37,13 +37,13 @@ services:
     volumes:
     - admin:/admin
     entrypoint:
-    - /bin/sh
+    - /bin/bash
     - -cex
     command:
     - |
       until test -f /admin/.kube/kubeconfig.yaml; do echo "Waiting for a kubeconfig ..." && sleep 1; done
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
-      kubectl-waitretry kubectl wait --for=condition=Ready node --all
+      kubectl-waitretry --for=condition=Ready node --all
       echo "Doing nothing ATM"
       tail -f /dev/null
 
diff --git a/runner.Dockerfile b/runner.Dockerfile
index 71d6a043..8be62c67 100644
--- a/runner.Dockerfile
+++ b/runner.Dockerfile
@@ -37,3 +37,4 @@ COPY bin/y-skaffold /usr/local/src/ystack/bin/
 RUN y-skaffold
 
 COPY . /usr/local/src/ystack
+WORKDIR /usr/local/src/ystack
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index b5160c2b..e43ba616 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -1,7 +1,7 @@
 *
 !bin/y-*
 bin/*-bin
-!bin/kubectl-
+!bin/kubectl-*
 
 # Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date:
 !converge-generic/

From 49c3514d07b32b579de1b20e8378b4aa17ef506e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 15:15:57 +0100
Subject: [PATCH 15/39] As much memory for k3s-docker that we had for
 k3s-multipass,

still supporting SCALE=2+ but with compose format 2.x to use daemon's resource flags
---
 docker-compose.dev-overrides.yml | 4 +++-
 docker-compose.test.yml          | 2 +-
 runner.Dockerfile.dockerignore   | 3 +++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index a25dcde4..9d0bac15 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -1,4 +1,4 @@
-version: '3'
+version: '2.4'
 services:
 
   server:
@@ -9,3 +9,5 @@ services:
     ports:
     - 80:30080
     - 443:30443
+    mem_limit: 8589934592
+    memswap_limit: 0
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 31a99113..eaf58bd3 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -1,4 +1,4 @@
-version: '3'
+version: '2.4'
 services:
 
   server:
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index e43ba616..806bd3b6 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -9,3 +9,6 @@ bin/*-bin
 !registry/generic/
 !registry/node-update-hosts/
 !registry/generic,minio/
+!buildkit/
+!monitoring/rbac-base/
+!monitoring/instance-now/

From 0334923cde5b6c3099a023a65dae5566e4291383 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 15:54:48 +0100
Subject: [PATCH 16/39] Adds registries conf through a docker(-compose) build
 step

---
 docker-compose.test.yml   | 4 ++--
 k3s/image/Dockerfile      | 3 +++
 k3s/image/registries.yaml | 7 +++++++
 3 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 k3s/image/Dockerfile
 create mode 100644 k3s/image/registries.yaml

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index eaf58bd3..513402ed 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -2,7 +2,7 @@ version: '2.4'
 services:
 
   server:
-    image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
+    build: ./k3s/image
     command: server --disable-agent
     environment:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
@@ -16,7 +16,7 @@ services:
     #- 6443:6443
 
   node:
-    image: rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
+    build: ./k3s/image
     tmpfs:
     - /run
     - /var/run
diff --git a/k3s/image/Dockerfile b/k3s/image/Dockerfile
new file mode 100644
index 00000000..ed0da34a
--- /dev/null
+++ b/k3s/image/Dockerfile
@@ -0,0 +1,3 @@
+FROM rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
+
+COPY registries.yaml /etc/rancher/k3s/registries.yaml
diff --git a/k3s/image/registries.yaml b/k3s/image/registries.yaml
new file mode 100644
index 00000000..423ef968
--- /dev/null
+++ b/k3s/image/registries.yaml
@@ -0,0 +1,7 @@
+mirrors:
+  "builds-registry.ystack.svc.cluster.local":
+    endpoint:
+    - http://builds-registry.ystack.svc.cluster.local
+  "prod-registry.ystack.svc.cluster.local":
+    endpoint:
+    - http://prod-registry.ystack.svc.cluster.local

From 627fecade757139a7cdfb906517eaaba2afdb9fa Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 15:55:32 +0100
Subject: [PATCH 17/39] Any provision impl should include namespace +
 prometheus-operator

---
 bin/y-cluster-provision-k3s-docker | 3 ---
 docker-compose.test.yml            | 4 +++-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/bin/y-cluster-provision-k3s-docker b/bin/y-cluster-provision-k3s-docker
index 3680c46c..c1eeb5e4 100755
--- a/bin/y-cluster-provision-k3s-docker
+++ b/bin/y-cluster-provision-k3s-docker
@@ -28,9 +28,6 @@ cat << EOF
 docker-compose -p ystack -f docker-compose.test.yml -f docker-compose.dev-overrides.yml exec server sh -c 'cat /admin/.kube/kubeconfig.yaml'
 export KUBECONFIG=$(pwd)/k.yaml
 
-kubectl create namespace ystack
-kubectl apply -k converge-generic/
-
 # nodePort should match port forward
 kubectl -n kube-system edit service traefik
 EOF
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 513402ed..c7a92f2b 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -44,7 +44,9 @@ services:
       until test -f /admin/.kube/kubeconfig.yaml; do echo "Waiting for a kubeconfig ..." && sleep 1; done
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
-      echo "Doing nothing ATM"
+      kubectl create namespace ystack
+      y-cluster-install-prometheus-operator
+      echo "Will stay running for manual work"
       tail -f /dev/null
 
 volumes:

From 4b88675cf8c8b32d25ae34f92d21d6da6e338a9b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 20:56:28 +0100
Subject: [PATCH 18/39] -k3s-docker is now equivalent to -k3s-multipass

---
 bin/y-cluster-provision-k3s-docker | 11 ++++-------
 docker-compose.test.yml            |  1 +
 runner.Dockerfile.dockerignore     |  1 +
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/bin/y-cluster-provision-k3s-docker b/bin/y-cluster-provision-k3s-docker
index c1eeb5e4..8289d4b3 100755
--- a/bin/y-cluster-provision-k3s-docker
+++ b/bin/y-cluster-provision-k3s-docker
@@ -22,12 +22,9 @@ false || {
   echo "# Currently, because sut doesn't wait for server to be up, you need another:" && echo "$COMPOSE up -d" && \
   true )
 
-cat << EOF
-# Misc notes
+$COMPOSE exec server sh -c 'until test -f /admin/.kube/kubeconfig.yaml; do echo "# Waiting ..." && sleep 1; done; cat /admin/.kube/kubeconfig.yaml' \
+  > "$KUBECONFIG.tmp"
 
-docker-compose -p ystack -f docker-compose.test.yml -f docker-compose.dev-overrides.yml exec server sh -c 'cat /admin/.kube/kubeconfig.yaml'
-export KUBECONFIG=$(pwd)/k.yaml
+KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local
 
-# nodePort should match port forward
-kubectl -n kube-system edit service traefik
-EOF
+y-kubeconfig-import "$KUBECONFIG.tmp"
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index c7a92f2b..99903f49 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -44,6 +44,7 @@ services:
       until test -f /admin/.kube/kubeconfig.yaml; do echo "Waiting for a kubeconfig ..." && sleep 1; done
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
+      kubectl apply -k ./metrics-server/
       kubectl create namespace ystack
       y-cluster-install-prometheus-operator
       echo "Will stay running for manual work"
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index 806bd3b6..53910b47 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -4,6 +4,7 @@ bin/*-bin
 !bin/kubectl-*
 
 # Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date:
+!metrics-server/
 !converge-generic/
 !minio/
 !registry/generic/

From 8881aa9df7f7ed77635d65d5370a93a7bc9d2566 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 21:00:37 +0100
Subject: [PATCH 19/39] I don't think we'll ned k3d

---
 bin/y-k3d | 17 -----------------
 1 file changed, 17 deletions(-)
 delete mode 100755 bin/y-k3d

diff --git a/bin/y-k3d b/bin/y-k3d
deleted file mode 100755
index c4a52a8f..00000000
--- a/bin/y-k3d
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/sh
-[ -z "$DEBUG" ] || set -x
-set -e
-
-version=1.3.4
-
-bin_name=k3d \
-  bin_version=v${version} \
-  Darwin_url=https://github.com/rancher/k3d/releases/download/v${version}/k3d-darwin-amd64 \
-  Darwin_sha256=011072c5be4f33af606873aa7e326246f2ee1a158c07ed4f2791eb2e3de184e3 \
-  Linux_url=https://github.com/rancher/k3d/releases/download/v${version}/k3d-linux-amd64 \
-  Linux_sha256=41e0579b1499f6051e9d0d8a6d39d5f419e1037851c0121b676190af5825f4c0 \
-  y-bin-dependency-download || exit $?
-
-echo "NOTE y-stack on k3s is quite blocked by https://github.com/rancher/k3d/issues/134"
-
-y-k3d-v${version}-bin "$@" || echo $?

From 50512adf4a7592b13c33710ae8456ce2dda6c133 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 27 Oct 2019 16:35:17 +0100
Subject: [PATCH 20/39] Prometheus Operator takes 2m+20M resources when idle,

and is quite impractical with Kustomize due to:

 - Large repo that takes a long time to fetch
 - Namespace "default" is hard coded in bundle.yaml

... so let's install it by default
---
 bin/y-cluster-install-prometheus-operator | 7 +++++++
 bin/y-cluster-provision-gke               | 2 ++
 bin/y-cluster-provision-k3s-multipass     | 2 ++
 3 files changed, 11 insertions(+)
 create mode 100755 bin/y-cluster-install-prometheus-operator

diff --git a/bin/y-cluster-install-prometheus-operator b/bin/y-cluster-install-prometheus-operator
new file mode 100755
index 00000000..3e1dd0d7
--- /dev/null
+++ b/bin/y-cluster-install-prometheus-operator
@@ -0,0 +1,7 @@
+#!/bin/sh
+[ -z "$DEBUG" ] || set -x
+set -e
+
+OPERATOR_VERSION=ab1f8da8d9195339266742dcef535df5b51522e1
+
+kubectl -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml
diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 11dcb7e2..4d206d26 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -137,3 +137,5 @@ EOF
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \
   echo "Warning: failed to create builds registry storage bucket"
+
+y-cluster-install-prometheus-operator
diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass
index 9b2063db..4c40862a 100755
--- a/bin/y-cluster-provision-k3s-multipass
+++ b/bin/y-cluster-provision-k3s-multipass
@@ -90,6 +90,8 @@ KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack
 
 KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server
 
+KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator
+
 y-kubeconfig-import "$KUBECONFIG.tmp"
 
 echo "# Done. Master IP: $K3S_NODEIP_MASTER. The cluster should be ready for y-stack converge now."

From 40177ce2d0326e2b742858d52c79172aaafbd1ef Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 21:11:32 +0100
Subject: [PATCH 21/39] Don't fail if restarted

---
 docker-compose.test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 99903f49..0eddb2fc 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -45,7 +45,7 @@ services:
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
       kubectl apply -k ./metrics-server/
-      kubectl create namespace ystack
+      kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack
       y-cluster-install-prometheus-operator
       echo "Will stay running for manual work"
       tail -f /dev/null

From dd39cc6a5d3ebdf2d82044f26b8fe18b59973915 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 21:17:33 +0100
Subject: [PATCH 22/39] Let the test exit, unless told to stay around

---
 docker-compose.dev-overrides.yml | 4 ++++
 docker-compose.test.yml          | 7 +++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index 9d0bac15..17666ca2 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -11,3 +11,7 @@ services:
     - 443:30443
     mem_limit: 8589934592
     memswap_limit: 0
+
+  sut:
+    environment:
+    - KEEP_RUNNING=true
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 0eddb2fc..2751bf87 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -34,6 +34,7 @@ services:
       dockerfile: runner.Dockerfile
     environment:
     - KUBECONFIG=/admin/.kube/config
+    - KEEP_RUNNING=false
     volumes:
     - admin:/admin
     entrypoint:
@@ -47,8 +48,10 @@ services:
       kubectl apply -k ./metrics-server/
       kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack
       y-cluster-install-prometheus-operator
-      echo "Will stay running for manual work"
-      tail -f /dev/null
+      if [ "$$KEEP_RUNNING" = "true" ]; then
+        echo "Will stay running for manual work"
+        sleep infinity
+      fi
 
 volumes:
   k3s-server: {}

From 1d66fa567eb35f7dd53bd7d4eed9a87eb0c0d878 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 11 Nov 2019 21:18:37 +0100
Subject: [PATCH 23/39] docker-compose doesn't use buildkit and therefore not
 the

Dockerfile-specific ignore. Let's hope that a symlink works with docker hub.
---
 .dockerignore | 1 +
 1 file changed, 1 insertion(+)
 create mode 120000 .dockerignore

diff --git a/.dockerignore b/.dockerignore
new file mode 120000
index 00000000..3f5a7b5b
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1 @@
+runner.Dockerfile.dockerignore
\ No newline at end of file

From 7f971218983c601500ac24281e126dd9abed0470 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 05:44:21 +0100
Subject: [PATCH 24/39] Docker Hub's automated test would only start "sut"

---
 docker-compose.test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 2751bf87..f4b99038 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -28,6 +28,9 @@ services:
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
 
   sut:
+    depends_on:
+    - server
+    - node
     build:
       context: .
       # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore

From 35c5603af7e73de0fbd09385beec1cf3b12a8e5e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 11:36:08 +0100
Subject: [PATCH 25/39] In docker hub's builds we never get a kubeconfig,

---
 docker-compose.test.yml |  8 +++++++-
 test.sh                 | 21 +++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)
 create mode 100755 test.sh

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index f4b99038..1102d30f 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -37,6 +37,7 @@ services:
       dockerfile: runner.Dockerfile
     environment:
     - KUBECONFIG=/admin/.kube/config
+    - KUBECONFIG_WAIT=30
     - KEEP_RUNNING=false
     volumes:
     - admin:/admin
@@ -45,7 +46,12 @@ services:
     - -cex
     command:
     - |
-      until test -f /admin/.kube/kubeconfig.yaml; do echo "Waiting for a kubeconfig ..." && sleep 1; done
+      until test -f /admin/.kube/kubeconfig.yaml; do
+        [ $$KUBECONFIG_WAIT -gt 0 ] || exit 1
+        KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 ))
+        echo "Waiting for a kubeconfig ..."
+        sleep 1
+      done
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
       kubectl apply -k ./metrics-server/
diff --git a/test.sh b/test.sh
new file mode 100755
index 00000000..69bc8d04
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# What we think Docker Hub is running
+docker-compose -f docker-compose.test.yml up sut
+RESULT=$?
+docker-compose -f docker-compose.test.yml down --remove-orphans -v
+
+cat << EOF
+# To get a docker environment similar(?) to docker hub's locally, on for example multipass ubuntu 18.04:
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+sudo add-apt-repository    "deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable"
+sudo apt-get update
+apt-cache madison docker-ce
+sudo apt-get install docker-ce=18.03.1~ce~3-0~ubuntu
+sudo usermod -aG docker multipass
+sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-\$(uname -s)-\$(uname -m)" -o /usr/local/bin/docker-compose
+sudo chmod +x /usr/local/bin/docker-compose
+EOF
+
+echo "Result: $RESULT"
+exit $RESULT

From 4450b39c280c0fc37705d8e15181982611987ed7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 11:48:47 +0100
Subject: [PATCH 26/39] Server's memory capped at <=128MB reproduces the Hub
 build issue

---
 docker-compose.test.yml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 1102d30f..28bf6137 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -14,6 +14,8 @@ services:
     # Host port not needed for CI
     #ports:
     #- 6443:6443
+    mem_limit: 268435456
+    memswap_limit: 0
 
   node:
     build: ./k3s/image
@@ -26,6 +28,8 @@ services:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
+    mem_limit: 1610612736
+    memswap_limit: 0
 
   sut:
     depends_on:
@@ -49,8 +53,7 @@ services:
       until test -f /admin/.kube/kubeconfig.yaml; do
         [ $$KUBECONFIG_WAIT -gt 0 ] || exit 1
         KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 ))
-        echo "Waiting for a kubeconfig ..."
-        sleep 1
+        echo "Waiting for a kubeconfig ..." && sleep 1
       done
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
@@ -61,6 +64,8 @@ services:
         echo "Will stay running for manual work"
         sleep infinity
       fi
+    mem_limit: 67108864
+    memswap_limit: 0
 
 volumes:
   k3s-server: {}

From f5a5d65c09f8b84d824299414515e40e900ef2d2 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 11:53:53 +0100
Subject: [PATCH 27/39] Makes it easier to see if we're under the 2GB limit

---
 docker-compose.test.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 28bf6137..07f83cc1 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -14,7 +14,7 @@ services:
     # Host port not needed for CI
     #ports:
     #- 6443:6443
-    mem_limit: 268435456
+    mem_limit:  300000000
     memswap_limit: 0
 
   node:
@@ -28,7 +28,7 @@ services:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
-    mem_limit: 1610612736
+    mem_limit: 1600000000
     memswap_limit: 0
 
   sut:
@@ -64,7 +64,7 @@ services:
         echo "Will stay running for manual work"
         sleep infinity
       fi
-    mem_limit: 67108864
+    mem_limit:   80000000
     memswap_limit: 0
 
 volumes:

From fa59d3ceb5c9b7bd4f74e66cbc582702f23fa4cf Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 11:55:34 +0100
Subject: [PATCH 28/39] Let's not push the memory limit, yet

---
 docker-compose.test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 07f83cc1..9889f5a9 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -28,7 +28,7 @@ services:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
-    mem_limit: 1600000000
+    mem_limit: 1500000000
     memswap_limit: 0
 
   sut:

From 1816b9e4d572582f54e8f0eb8a3cd299e75eb900 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 12:03:52 +0100
Subject: [PATCH 29/39] Temporarily run the server as SUT so we get logs

---
 docker-compose.test.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 9889f5a9..d6e30e99 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -1,7 +1,9 @@
 version: '2.4'
 services:
 
-  server:
+  sut:
+    depends_on:
+    - node
     build: ./k3s/image
     command: server --disable-agent
     environment:
@@ -24,16 +26,16 @@ services:
     - /var/run
     privileged: true
     environment:
-    - K3S_URL=https://server:6443
+    - K3S_URL=https://sut:6443
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
     mem_limit: 1500000000
     memswap_limit: 0
 
-  sut:
+  runner:
     depends_on:
-    - server
+    - sut
     - node
     build:
       context: .

From dbe239e7ce14de7b54f73763ee1064e0c103e649 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 12:21:36 +0100
Subject: [PATCH 30/39] Maybe privileged mode is disallowed

---
 docker-compose.test.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index d6e30e99..4493c270 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -24,7 +24,6 @@ services:
     tmpfs:
     - /run
     - /var/run
-    privileged: true
     environment:
     - K3S_URL=https://sut:6443
     - K3S_CLUSTER_SECRET=somethingtotallyrandom

From 7666da5e04db76251452148b2898a772542036d1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 12:28:14 +0100
Subject: [PATCH 31/39] Nodes don't start when non-privileged

This reverts commit dbe239e7ce14de7b54f73763ee1064e0c103e649.
---
 docker-compose.test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 4493c270..d6e30e99 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -24,6 +24,7 @@ services:
     tmpfs:
     - /run
     - /var/run
+    privileged: true
     environment:
     - K3S_URL=https://sut:6443
     - K3S_CLUSTER_SECRET=somethingtotallyrandom

From 575e968e5042500f4d483b5dbc4a1a2f19dae6d3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 12:30:38 +0100
Subject: [PATCH 32/39] Hub builds die after "Serving healthz insecurely on"

---
 docker-compose.test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index d6e30e99..5a526c9e 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -5,7 +5,7 @@ services:
     depends_on:
     - node
     build: ./k3s/image
-    command: server --disable-agent
+    command: server --disable-agent -v=7
     environment:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml

From f656c50007f1c1d67aaa661c5eeff02027349322 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 12:41:18 +0100
Subject: [PATCH 33/39] Uses an actual array for entrypoint

---
 docker-compose.test.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 5a526c9e..bf7729d7 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -5,7 +5,10 @@ services:
     depends_on:
     - node
     build: ./k3s/image
-    command: server --disable-agent -v=7
+    command:
+    - server
+    - --disable-agent
+    - -v=7
     environment:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml

From 7ea0e02c540a6b13f62030520976cd46952ebf03 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 12:41:45 +0100
Subject: [PATCH 34/39] Hub builds seem to simply kill sever arbitrarily

---
 docker-compose.test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index bf7729d7..84574d92 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -19,7 +19,7 @@ services:
     # Host port not needed for CI
     #ports:
     #- 6443:6443
-    mem_limit:  300000000
+    mem_limit:  500000000
     memswap_limit: 0
 
   node:
@@ -33,7 +33,7 @@ services:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
-    mem_limit: 1500000000
+    mem_limit: 1300000000
     memswap_limit: 0
 
   runner:

From 75ac74e99a3019a7ebd79023eb4948e4842baffe Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 13:06:24 +0100
Subject: [PATCH 35/39] Got a timeout:

level=fatal msg="starting tls server: Get https://127.0.0.1:6444/apis/apiextensions.k8s.io/v1beta1/customresourcedefinitions: net/http: TLS handshake timeout"
---
 docker-compose.test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 84574d92..65e4cacd 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -19,7 +19,7 @@ services:
     # Host port not needed for CI
     #ports:
     #- 6443:6443
-    mem_limit:  500000000
+    mem_limit:  700000000
     memswap_limit: 0
 
   node:
@@ -33,7 +33,7 @@ services:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
-    mem_limit: 1300000000
+    mem_limit: 1000000000
     memswap_limit: 0
 
   runner:

From 84eaa7c1f1a33c6600dc865cf000f6bd7c2cb8a6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 13:19:36 +0100
Subject: [PATCH 36/39] Revert "Temporarily run the server as SUT so we get
 logs", giving up

on docker hub's test runs for now.

This reverts commit 1816b9e4d572582f54e8f0eb8a3cd299e75eb900.
---
 docker-compose.test.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 65e4cacd..4207c339 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -1,9 +1,7 @@
 version: '2.4'
 services:
 
-  sut:
-    depends_on:
-    - node
+  server:
     build: ./k3s/image
     command:
     - server
@@ -29,16 +27,16 @@ services:
     - /var/run
     privileged: true
     environment:
-    - K3S_URL=https://sut:6443
+    - K3S_URL=https://server:6443
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
     mem_limit: 1000000000
     memswap_limit: 0
 
-  runner:
+  sut:
     depends_on:
-    - sut
+    - server
     - node
     build:
       context: .

From 2565b315e5f26c678af6d089529d75efef332bc7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 13:20:53 +0100
Subject: [PATCH 37/39] Restores resource limits to the original guesses

---
 docker-compose.test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 4207c339..5213bb65 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -17,7 +17,7 @@ services:
     # Host port not needed for CI
     #ports:
     #- 6443:6443
-    mem_limit:  700000000
+    mem_limit:  300000000
     memswap_limit: 0
 
   node:
@@ -31,7 +31,7 @@ services:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
     #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
-    mem_limit: 1000000000
+    mem_limit: 1500000000
     memswap_limit: 0
 
   sut:

From 824e4a7886589cd7db748fd5c51d7e3fa3a36fb8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 13:31:40 +0100
Subject: [PATCH 38/39] Restores logging too

---
 docker-compose.test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 5213bb65..a2e3808f 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -6,7 +6,7 @@ services:
     command:
     - server
     - --disable-agent
-    - -v=7
+    - -v=0
     environment:
     - K3S_CLUSTER_SECRET=somethingtotallyrandom
     - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml

From 7fc11dee36685f8fa377d397d51ca327bf91b7c5 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 12 Nov 2019 13:41:31 +0100
Subject: [PATCH 39/39] Allow docker hub builds to pass without testing
 anything,

so we get a runner image.
We can use ./test.sh elsewhere to get CI-ish.
---
 docker-compose.test.yml |  2 +-
 test.sh                 | 14 +-------------
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index a2e3808f..abd38fb5 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -54,7 +54,7 @@ services:
     command:
     - |
       until test -f /admin/.kube/kubeconfig.yaml; do
-        [ $$KUBECONFIG_WAIT -gt 0 ] || exit 1
+        [ $$KUBECONFIG_WAIT -gt 0 ] || exit ${BULID_EXIT_CODE_ON_NO_CLUSTER:-0}
         KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 ))
         echo "Waiting for a kubeconfig ..." && sleep 1
       done
diff --git a/test.sh b/test.sh
index 69bc8d04..354992ea 100755
--- a/test.sh
+++ b/test.sh
@@ -1,21 +1,9 @@
 #!/bin/bash
 
 # What we think Docker Hub is running
-docker-compose -f docker-compose.test.yml up sut
+BULID_EXIT_CODE_ON_NO_CLUSTER=1 docker-compose -f docker-compose.test.yml up --exit-code-from sut --scale node=1 sut
 RESULT=$?
 docker-compose -f docker-compose.test.yml down --remove-orphans -v
 
-cat << EOF
-# To get a docker environment similar(?) to docker hub's locally, on for example multipass ubuntu 18.04:
-curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
-sudo add-apt-repository    "deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable"
-sudo apt-get update
-apt-cache madison docker-ce
-sudo apt-get install docker-ce=18.03.1~ce~3-0~ubuntu
-sudo usermod -aG docker multipass
-sudo curl -L "https://github.com/docker/compose/releases/download/1.24.1/docker-compose-\$(uname -s)-\$(uname -m)" -o /usr/local/bin/docker-compose
-sudo chmod +x /usr/local/bin/docker-compose
-EOF
-
 echo "Result: $RESULT"
 exit $RESULT