Essential ystack services' test coverage using y-assert

.editorconfig

@@ -0,0 +1,7 @@
+root = true
+
+[*]
+charset = utf-8
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2

README.md

@@ -97,11 +97,32 @@ docker volume rm ystack_admin 2> /dev/null || true
 
 ## Development
 
+Using the [y-docker-compose](./bin/y-docker-compose) wrapper that extends [docker-compose.test.yml](./docker-compose.test.yml) that is used for CI with [docker-compose.dev-overrides.yml](./docker-compose.dev-overrides.yml). The k3s [image](./k3s/docker-image/) is the stock k3s image with y-stack's local registry config.
+
+```
+y-docker-compose down
+y-docker-compose up --build -d master1
+y-docker-compose up --build -d ystack-proxy
+kubectl --kubeconfig=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml config rename-context default ystack-local
+y-kubie ctx -f $YSTACK_HOME/devcluster/.kube/kubeconfig.yaml
+```
+
+To add monitoring support run `y-cluster-assert-install`.
+
+For [dev loops](./examples/) and `y-assert` the docker stack replaces `y-kubefwd` (hard to use in CI)
+with container ports.
+You need `cat /etc/hosts | grep 127.0.0 | grep cluster.local` to have something like:
 ```
-compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
-$compose down \
-  ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
-sudo rm test/.kube/kubeconfig.yaml
-$compose up --build -d ystack-proxy
-export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
+127.0.0.1	builds-registry.ystack.svc.cluster.local
+127.0.0.1	buildkitd.ystack.svc.cluster.local
+127.0.0.1	monitoring.ystack.svc.cluster.local
 ```
+
+Test using:
+```
+curl http://builds-registry.ystack.svc.cluster.local/v2/
+curl http://monitoring.ystack.svc.cluster.local:9090/api/v1/alertmanagers | jq '.data.activeAlertmanagers[0]'
+curl http://monitoring.ystack.svc.cluster.local:9093/api/v2/status
+```
+
+Start a dev loop for actual asserts using `cd specs; y-skaffold --cache-artifacts=false dev` and start editing specs/*.spec.js. Run `y-assert` for CI-like runs until completion.

bin/.gitignore

@@ -2,9 +2,14 @@
 
 # our executables are symlinked to real names
 buildctl
+container-structure-test
+crane
 helm
+kubie
+kustomize
 minikube
 mkcert
+promtool
 rio
 skaffold
 kubefwd

bin/y-assert

@@ -22,6 +22,11 @@ using ephemeral namespaces.
 # What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor
 TTL="janitor/ttl: 23h"
 
+[ -z "$MONITORING_HOST" ] && MONITORING_HOST=http://monitoring.ystack.svc.cluster.local
+
+! curl -f -s --connect-timeout 3 $MONITORING_HOST:9090/ >/dev/null && echo "Failed to access the Prometheus endpoint" && exit 1
+! curl -f -s --connect-timeout 3 $MONITORING_HOST:9093/ >/dev/null && echo "Failed to access the Alertmanager endpoint" && exit 1
+
 ctx=$1
 case $ctx in
   "--context="*) shift 1 ;;
@@ -32,6 +37,9 @@ case $ctx in
 esac
 skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|')
 
+[ -z "$PROFILES" ] && PROFILES="y-assert"
+skaffoldflags="$skaffoldflags -p $PROFILES"
+
 CIRUN=$1
 [ -z "$CIRUN" ] && CIRUN="run"
 
@@ -55,7 +63,21 @@ EOF
 fi
 skaffoldflags="$skaffoldflags -n $namespace"
 
-set -x
+# At this point we should know how to identify relevant assert metrics
+[ -z "$assertlabels" ] && assertlabels="namespace=\"$namespace\"";
+
 y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN
 
+echo "$(log) Prometheus queries for test results will use labels: $assertlabels"
 echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: */=/') -o name | xargs kubectl $ctx delete"
+
+query=assertions_failed{$assertlabels}
+
+[ -z "$INTERVAL" ] && INTERVAL=5
+# This loop will continuously list errors and must fail on http errors or absence of the metric
+until curl -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \
+  | jq -r '.data.result | if length > 0 then . else error("No results") end | .[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end'
+do sleep $INTERVAL
+done
+
+echo "Zero failures now with: curl -s --data query=$query $MONITORING_HOST:9090/api/v1/query"

bin/y-build-like-sync

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+# Bash isn't a good tool for producing a tar from a build context
+# but this script is an experiment on how to compose an image
+# from a runtime and some static files using go-containerregistry's crane
+# The user shouldn't need to write Dockerfile or .dockerignore,
+# but we could probably use a generated Dockerfile + kaniko instead, or Buildpacks
+
+# Settings
+DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local
+[ -z "$BUILDS_REGISTRY" ] && BUILDS_REGISTRY=$DEFAULT_REGISTRY
+[ -z "$PUSH_REGISTRY" ]   && PUSH_REGISTRY=$DEFAULT_REGISTRY
+
+if [ "$(curl -s --connect-timeout 3 http://$BUILDS_REGISTRY/v2/)" != "{}" ]
+then
+  echo "ERROR Skaffold need local access to the builds registry for digest lookup"
+  echo "Registry: $BUILDS_REGISTRY"
+  echo "Look for y-stack's ingress or port-forward utilities"
+  exit 1
+fi
+
+[ -z "$IMAGE" ] && echo "No IMAGE env (from for example Skaffold)" && exit 1
+
+IMAGE=$IMAGE
+case "$IMAGE" in
+  $BUILDS_REGISTRY/* ) ;;
+  $PUSH_REGISTRY/* ) echo "Unlike y-build this script won't push to non-build registries" && exit 1 ;;
+  * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
+esac
+
+RUNTIME_IMAGE=$1
+[ -z "$RUNTIME_IMAGE" ] && echo "First argument must be a runtime image to append the layer to" \
+  && echo "To improve build times use a runtime image in the target repo" && exit 1
+
+# crane hangs for a long time if it doesn't know that the registry is plain http
+RUNTIME_IMAGE=$(echo $RUNTIME_IMAGE | sed 's|.local/|.local:80/|')
+IMAGE=$(echo $IMAGE | sed 's|.local/|.local:80/|')
+
+# This is a PoC, let's make a lot of assumptions to simplify
+context=.
+src='**'
+# assuming a single manual sync, which is the reasonable use case for a runtime
+[ ! -f skaffold.yaml ] && echo "This composition example assumes a sync defined in a skaffold.yaml" && exit 1
+dest=$(cat skaffold.yaml | grep 'dest:' | awk '{ print $2 }')
+# this avoids "tar: Removing leading `/' from member names" and could come in handy if we can't use --transform
+dest=$(echo $dest | sed 's|^/||')
+
+list=$(mktemp)
+(cd $context; git ls-files -c -o --exclude-standard -- . || find . -type f) > $list
+tar=$(mktemp)
+tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list --mode='ug+rw' --group=65534 --owner=65532
+rm $list
+
+set -x
+y-crane append --insecure -b $RUNTIME_IMAGE -f $tar -t $IMAGE
+set +x
+rm $tar

bin/y-build-nothing

@@ -24,20 +24,13 @@ case "$IMAGE" in
   * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
 esac
 
-[ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx"
-
-CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9
 NOTHING_IMAGE=busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209
 
-BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz")
-
 # TODO copy is significantly faster if the source image is already in the target repo
 
 # crane hangs for a long time if it doesn't know that the registry is plain http
 NOTHING_IMAGE=$(echo $NOTHING_IMAGE | sed 's|.local|.local:80|')
 IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
 
 set -x
-kubectl $ctx -n ystack run --restart=Never --attach \
-  --image=$CRANE_IMAGE y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") -- \
-  cp $NOTHING_IMAGE $IMAGE
+y-crane cp $NOTHING_IMAGE $IMAGE

bin/y-cluster-assert-install

@@ -0,0 +1,23 @@
+#!/bin/sh
+[ -z "$DEBUG" ] || set -x
+set -e
+
+OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
+KUBERNETES_ASSERT_VERSION=41207ccf8536ce022919d9aef86d2124b2c142b1
+
+ctx=$1
+case $ctx in
+  "--context="*) shift 1 ;;
+  *) echo "Initial arg must be --context=" && exit 1 ;;
+esac
+
+# If we're to make this script idempotent we must make sure that re-applying the operator's bundle isn't painfully slow
+kubectl $ctx create namespace monitoring
+
+kubectl $ctx -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml
+
+kubectl-waitretry $ctx -n default --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-operator
+
+kubectl $ctx -n monitoring apply -k github.com/Yolean/kubernetes-assert/example-small?ref=$KUBERNETES_ASSERT_VERSION
+
+kubectl-waitretry $ctx -n monitoring --for=condition=Ready pod --all

bin/y-cluster-provision-gke

@@ -94,19 +94,20 @@ esac
 
 gcloud container clusters create "$NAME" $FLAGS "$@"
 
-GKE_CONTEXT_NAME=$(kubectl config current-context)
+GKE_CONTEXT_NAME=$(kubectl $ctx config current-context)
+ctx="--context=${GKE_CONTEXT_NAME}"
 
 echo "For outbound Internet access: y-cluster-provision-gke-network-outbound-enable"
 
-kubectl create namespace ystack
+kubectl $ctx create namespace ystack
 
-kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
+kubectl $ctx -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
 
 ### To be moved to registry provision script, but we do want the initial context name
 
 [ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
 
-kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
+kubectl $ctx create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
 
 gcloud compute backend-buckets list
 gsutil ls
@@ -120,8 +121,8 @@ metadata:
 spec: {}
 EOF
 
-kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
+kubectl $ctx apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \
   echo "Warning: failed to create builds registry storage bucket"
 
-y-cluster-install-prometheus-operator
+y-cluster-assert-install $ctx

bin/y-cluster-provision-k3s-multipass

@@ -26,11 +26,11 @@ multipass exec "$VM_NAME" -- sudo cat /etc/rancher/k3s/k3s.yaml \
 
 KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local
 
-KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack
+KUBECONFIG="$KUBECONFIG.tmp" kubectl --context=local create namespace ystack
 
 #KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server
 
-KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator
+KUBECONFIG="$KUBECONFIG.tmp" y-cluster-assert-install --context=local
 
 y-kubeconfig-import "$KUBECONFIG.tmp"
 

bin/y-container-structure-test

@@ -12,4 +12,4 @@ bin_name=container-structure-test \
   Linux_sha256=09cb373a9e622a4120577391acb28c6b2aea55f883a4351db8e8de85f0cf8a5b \
   y-bin-dependency-download || exit $?
 
-y-container-structure-test-v${version}-bin "$@" || echo $?
+y-container-structure-test-v${version}-bin "$@" || exit $?

bin/y-crane

@@ -0,0 +1,17 @@
+#!/bin/sh
+[ -z "$DEBUG" ] || set -x
+set -e
+YBIN="$(dirname $0)"
+
+version=0.1.0
+
+bin_name=crane \
+  bin_version=v${version} \
+  Darwin_url=https://github.com/google/go-containerregistry/releases/download/v${version}/go-containerregistry_${version}_Darwin_x86_64.tar.gz \
+  Darwin_sha256=4bef595a9d0b6781b452b302317f6b70c72e43cce664f817fa777ff72f9ffe5f \
+  Linux_url=https://github.com/google/go-containerregistry/releases/download/v${version}/go-containerregistry_${version}_Linux_x86_64.tar.gz \
+  Linux_sha256=ee5bb44cc9097835d12e6b34e1c1e3399efb8ed3dfb68ee0f28c6e66952dc335 \
+  bin_tgz_path=crane \
+  $YBIN/y-bin-dependency-download || exit $?
+
+$YBIN/y-crane-v${version}-bin "$@" || exit $?

bin/y-docker-compose

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+[ -z "$YSTACK_HOME" ] && echo "Missing an YSTACK_HOME env" && exit 1
+
+COMPOSEFILES="-f $YSTACK_HOME/docker-compose.test.yml -f $YSTACK_HOME/docker-compose.dev-overrides.yml $COMPOSEFILES"
+
+KUBECONFIG=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml
+
+[ "$1" = "kubeconfig-path" ] && \
+  echo "$KUBECONFIG" && \
+  exit 0
+
+[ "$1" = "kubie" ] && \
+  y-kubie ctx -f $KUBECONFIG && \
+  exit 0
+
+docker-compose $COMPOSEFILES "$@"
+
+[ "$1" = "down" ] && \
+  echo "Removing local config" && \
+  (docker-compose $COMPOSEFILES up cleanup) && \
+  echo "Cleaning up volumes" && \
+  (docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true) && \
+  echo "Done"

bin/y-kubefwd

@@ -3,14 +3,14 @@
 set -e
 YBIN="$(dirname $0)"
 
-version=1.13.0
+version=1.14.0
 
 bin_name=kubefwd \
   bin_version=v${version} \
   Darwin_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_Darwin_amd64.tar.gz \
-  Darwin_sha256=49bce5f5914d81a478e301fae79ca63698b6eeaffcb38cf80bfc19150bf141e3 \
+  Darwin_sha256=1cb03198526192f21d632d531575ed91f6e7a58b5897bdb5814ed524c5e3d607 \
   Linux_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_linux_amd64.tar.gz \
-  Linux_sha256=4a6bc0f33bc0cde8738cc9611154702499d9581f3d1c0e65fb2d3951c1154ea5 \
+  Linux_sha256=f202a030aeaa6f9c5e0e6b764a0426a309662e81bab33505296c07391fc62b40 \
   bin_tgz_path=kubefwd \
   $YBIN/y-bin-dependency-download || exit $?
 

bin/y-kubie

@@ -16,4 +16,4 @@ command -v fzf > /dev/null || echo "Note that kubie recommends having fzf instal
 
 [ "$1" = "completion" ] && curl -sLS https://github.com/sbstp/kubie/raw/3ebbf728cc52a57dfd9913da120d47a44e783702/completion/kubie.bash && exit 0
 
-y-kubie-v${version}-bin "$@" || echo $?
+y-kubie-v${version}-bin "$@" || exit $?

bin/y-kustomize

@@ -12,4 +12,4 @@ bin_name=kustomize \
   Linux_sha256=a04d79a013827c9ebb0abfe9d41cbcedf507a0310386c8d9a7efec7a36f9d7a3 \
   y-bin-dependency-download || exit $?
 
-y-kustomize-v${version}-bin "$@" || echo $?
+y-kustomize-v${version}-bin "$@" || exit $?

bin/y-minikube

@@ -11,4 +11,4 @@ bin_name=minikube \
   Darwin_sha256=5bda29e2d990bb8ac9da1767143e228772adc45507d22a49b5af70b03e7db682 \
   y-bin-dependency-download || exit $?
 
-y-minikube-v${version}-bin "$@" || echo $?
+y-minikube-v${version}-bin "$@" || exit $?

bin/y-skaffold

@@ -28,4 +28,4 @@ grep 'current-context: minikube' $KUBECONFIG && {
 SKAFFOLD_NO_PRUNE=true \
 SKAFFOLD_UPDATE_CHECK=false \
   \
-  $YBIN/y-skaffold-v${version}-bin "$@" || echo $?
+  $YBIN/y-skaffold-v${version}-bin "$@" || exit $?