Automate some sanity checks

.dockerignore

@@ -0,0 +1 @@
+runner.Dockerfile.dockerignore

bin/kubectl-waitretry

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+[ -z "$WAITRETRIES" ] && WAITRETRIES=10
+
+SLEEP=1
+RETRY=0
+until kubectl wait $@; do
+  [ $RETRY -lt $WAITRETRIES ] || exit 1
+  echo "waitretry after $SLEEP seconds ..."
+  sleep $SLEEP
+  SLEEP=$(( $SLEEP * 2 ))
+  RETRY=$(( $RETRY + 1 ))
+done

bin/y-cluster-install-prometheus-operator

@@ -0,0 +1,7 @@
+#!/bin/sh
+[ -z "$DEBUG" ] || set -x
+set -e
+
+OPERATOR_VERSION=ab1f8da8d9195339266742dcef535df5b51522e1
+
+kubectl -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml

bin/y-cluster-provision-gke

@@ -137,3 +137,5 @@ EOF
 kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \
   echo "Warning: failed to create builds registry storage bucket"
+
+y-cluster-install-prometheus-operator

bin/y-cluster-provision-k3s-docker

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+[ -z "$KUBECONFIG" ] && echo "Provision requires an explicit KUBECONFIG env" && exit 1
+
+COMPOSE="docker-compose -p ystack -f docker-compose.test.yml"
+[ -z "$NODES" ] && NODES=2
+# TODO condition to toggle dev/qa mode
+false || {
+  COMPOSE="$COMPOSE -f docker-compose.dev-overrides.yml"
+  # ATM with port-forward we only support one node
+  [ $NODES -eq 1 ] || echo "# Dev mode only supports one node, util we add some kind of load balancer container"
+  NODES=1
+}
+
+(cd $YSTACK_HOME/ && \
+  $COMPOSE up --scale node=$NODES -d --build && \
+  docker ps && \
+  echo "# Compose commmand:" && echo "$COMPOSE" && \
+  echo "# To tear down:" && echo "$COMPOSE down --remove-orphans -v" && \
+  echo "# Currently, because sut doesn't wait for server to be up, you need another:" && echo "$COMPOSE up -d" && \
+  true )
+
+$COMPOSE exec server sh -c 'until test -f /admin/.kube/kubeconfig.yaml; do echo "# Waiting ..." && sleep 1; done; cat /admin/.kube/kubeconfig.yaml' \
+  > "$KUBECONFIG.tmp"
+
+KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local
+
+y-kubeconfig-import "$KUBECONFIG.tmp"

bin/y-cluster-provision-k3s-multipass

@@ -90,6 +90,8 @@ KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack
 
 KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server
 
+KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator
+
 y-kubeconfig-import "$KUBECONFIG.tmp"
 
 echo "# Done. Master IP: $K3S_NODEIP_MASTER. The cluster should be ready for y-stack converge now."

bin/y-kubefwd

@@ -3,12 +3,14 @@
 set -e
 YBIN="$(dirname $0)"
 
-version=1.9.2
+version=1.9.6
 
 bin_name=kubefwd \
   bin_version=v${version} \
-  Darwin_url=https://github.com/txn2/kubefwd/releases/download/v${version}/kubefwd_macOS_amd64.tar.gz \
-  Darwin_sha256=3f49349f7de2c8f60ae4d1248be4f936c9082d2b23cf9fa093caef20a60685d4 \
+  Darwin_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_Darwin_amd64.tar.gz \
+  Darwin_sha256=8a9a75d5377fbc8fc83152cdab4319b0e72f71ef084ef07703713b63e3ca3388 \
+  Linux_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_linux_amd64.tar.gz \
+  Linux_sha256=4a37c2f9763b46d2aba25cb6d07be36775213e208525911df27e398ea7d94d31 \
   bin_tgz_path=kubefwd \
   y-bin-dependency-download || exit $?
 

bin/y-skaffold

@@ -5,14 +5,14 @@ YBIN="$(dirname $0)"
 
 [ "$1" = "dev" ] && echo "y-skaffold dev requires -n [namespace] as initial args" && exit 1
 
-version=0.40.0
+version=1.0.0
 
 bin_name=skaffold \
   bin_version=v${version} \
   Darwin_url=https://github.com/GoogleContainerTools/skaffold/releases/download/v${version}/skaffold-darwin-amd64 \
-  Darwin_sha256=5175e11b36009cf5c1ba53f1564a2c1e66fd5a48a87e73a9bb672bade6c51174 \
+  Darwin_sha256=5a7b4dad8dfa83129c2076e799fa5a1a004f8c40cae0bb024828d5b542a96a84 \
   Linux_url=https://github.com/GoogleContainerTools/skaffold/releases/download/v${version}/skaffold-linux-amd64 \
-  Linux_sha256=f2d876e60cfc0dfb0fc79263bc9900e9c08a8300bf1e4b2bbfef3f9936cc78e0 \
+  Linux_sha256=f03cb2115cd510b6de8b024165b6621e453935faaf99b20c8cdf9933eec9b913 \
   y-bin-dependency-download || exit $?
 
 # https://skaffold.dev/docs/concepts/#local-development

docker-compose.dev-overrides.yml

@@ -0,0 +1,17 @@
+version: '2.4'
+services:
+
+  server:
+    ports:
+    - 6443:6443
+
+  node:
+    ports:
+    - 80:30080
+    - 443:30443
+    mem_limit: 8589934592
+    memswap_limit: 0
+
+  sut:
+    environment:
+    - KEEP_RUNNING=true

docker-compose.test.yml

@@ -0,0 +1,75 @@
+version: '2.4'
+services:
+
+  server:
+    build: ./k3s/image
+    command:
+    - server
+    - --disable-agent
+    - -v=0
+    environment:
+    - K3S_CLUSTER_SECRET=somethingtotallyrandom
+    - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml
+    - K3S_KUBECONFIG_MODE=666
+    volumes:
+    - k3s-server:/var/lib/rancher/k3s
+    - admin:/admin
+    # Host port not needed for CI
+    #ports:
+    #- 6443:6443
+    mem_limit:  300000000
+    memswap_limit: 0
+
+  node:
+    build: ./k3s/image
+    tmpfs:
+    - /run
+    - /var/run
+    privileged: true
+    environment:
+    - K3S_URL=https://server:6443
+    - K3S_CLUSTER_SECRET=somethingtotallyrandom
+    # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
+    #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
+    mem_limit: 1500000000
+    memswap_limit: 0
+
+  sut:
+    depends_on:
+    - server
+    - node
+    build:
+      context: .
+      # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore
+      dockerfile: runner.Dockerfile
+    environment:
+    - KUBECONFIG=/admin/.kube/config
+    - KUBECONFIG_WAIT=30
+    - KEEP_RUNNING=false
+    volumes:
+    - admin:/admin
+    entrypoint:
+    - /bin/bash
+    - -cex
+    command:
+    - |
+      until test -f /admin/.kube/kubeconfig.yaml; do
+        [ $$KUBECONFIG_WAIT -gt 0 ] || exit ${BULID_EXIT_CODE_ON_NO_CLUSTER:-0}
+        KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 ))
+        echo "Waiting for a kubeconfig ..." && sleep 1
+      done
+      cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config
+      kubectl-waitretry --for=condition=Ready node --all
+      kubectl apply -k ./metrics-server/
+      kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack
+      y-cluster-install-prometheus-operator
+      if [ "$$KEEP_RUNNING" = "true" ]; then
+        echo "Will stay running for manual work"
+        sleep infinity
+      fi
+    mem_limit:   80000000
+    memswap_limit: 0
+
+volumes:
+  k3s-server: {}
+  admin: {}

docker/dockerd-statefulset.yaml

@@ -32,7 +32,7 @@ spec:
           ln -s /var/lib/docker/overlay2/var-lib-docker-image /var/lib/docker/image
       containers:
       - name: dockerd
-        image: docker:19.03.3-dind@sha256:5d11c2a1300b7438e21cbb5b2bd704e663b70534abe90c9072081dfb18e144de
+        image: docker:19.03.4-dind@sha256:156e4e723c80223298259dad9a925a71f385456b731c139bec12d29b3a74170c
         securityContext:
           privileged: true
         args:

examples/basic-dev-inner-loop/skaffold.yaml

@@ -1,4 +1,4 @@
-apiVersion: skaffold/v1beta16
+apiVersion: skaffold/v1
 kind: Config
 build:
   tagPolicy:

k3s/image/Dockerfile

@@ -0,0 +1,3 @@
+FROM rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db
+
+COPY registries.yaml /etc/rancher/k3s/registries.yaml

k3s/image/registries.yaml

@@ -0,0 +1,7 @@
+mirrors:
+  "builds-registry.ystack.svc.cluster.local":
+    endpoint:
+    - http://builds-registry.ystack.svc.cluster.local
+  "prod-registry.ystack.svc.cluster.local":
+    endpoint:
+    - http://prod-registry.ystack.svc.cluster.local

runner.Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:19.10@sha256:a21f154506cc00974f647e13dbba6b7035da35c7669a4bb919515d895221face
+FROM ubuntu:19.10@sha256:a5193c15d7705bc2be91781355c4932321c06c18914facdd113d5bfcace7f92d
 
 RUN set -ex; \
   export DEBIAN_FRONTEND=noninteractive; \
@@ -12,10 +12,10 @@ RUN set -ex; \
 
 RUN set -ex; \
   F=$(mktemp); \
-  curl -SLs https://dl.k8s.io/v1.16.1/kubernetes-client-linux-amd64.tar.gz \
+  curl -SLs https://dl.k8s.io/v1.16.2/kubernetes-client-linux-amd64.tar.gz \
     | tee $F \
     | tar xzf - --strip-components=3 -C /usr/local/bin/; \
-  echo "e355a74a17d96785b0b217673e67fa0f02daa1939f10d410602ac0a0d061a4db71d727b67f75aa886007dab95dd5c7f8cc38253d291dc4d2504ce673df69fb32 $F" \
+  echo "69bb92c9b16c0286d7401d87cc73b85c88d6f9a17d2cf1748060e44525194b5be860daf7554c4c6319a546c9ff10f2b4df42a27e32d8f95a4052993b17ef57c0 $F" \
     | sha512sum -c -; \
   rm $F
 
@@ -37,3 +37,4 @@ COPY bin/y-skaffold /usr/local/src/ystack/bin/
 RUN y-skaffold
 
 COPY . /usr/local/src/ystack
+WORKDIR /usr/local/src/ystack

runner.Dockerfile.dockerignore

@@ -1,3 +1,15 @@
 *
 !bin/y-*
 bin/*-bin
+!bin/kubectl-*
+
+# Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date:
+!metrics-server/
+!converge-generic/
+!minio/
+!registry/generic/
+!registry/node-update-hosts/
+!registry/generic,minio/
+!buildkit/
+!monitoring/rbac-base/
+!monitoring/instance-now/

test.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# What we think Docker Hub is running
+BULID_EXIT_CODE_ON_NO_CLUSTER=1 docker-compose -f docker-compose.test.yml up --exit-code-from sut --scale node=1 sut
+RESULT=$?
+docker-compose -f docker-compose.test.yml down --remove-orphans -v
+
+echo "Result: $RESULT"
+exit $RESULT