From 861590c531cb13435c1e4ecf3fef96e1eb0d641e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 12:41:37 +0200
Subject: [PATCH 01/61] CI should validate our examples, and run our specs

---
 README.md               |  3 +++
 docker-compose.test.yml | 26 +++++++++++++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 4242b59f..4ac95705 100644
--- a/README.md
+++ b/README.md
@@ -105,3 +105,6 @@ sudo rm test/.kube/kubeconfig.yaml
 $compose up --build -d ystack-proxy
 export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
 ```
+
+With kubectl access to the in-docker cluster you might want to start with
+some of the steps from the "sut" script in [docker-compose.test.yml](./docker-compose.test.yml).
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index dfa958f6..a0c2b06d 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -77,6 +77,7 @@ services:
     environment:
     - KUBECONFIG_WAIT=30
     - KEEP_RUNNING=false
+    - EXAMPLES=sync-only in-cluster-build basic-dev-inner-loop
     volumes:
     - admin:/admin
     - ./examples:/usr/local/src/ystack/examples
@@ -85,6 +86,8 @@ services:
     - -cx
     command:
     - |
+      [ "$$KEEP_RUNNING" = "true" ] || set -e
+
       mkdir ~/.kube
       until test -f /admin/.kube/kubeconfig.yaml; do
         [ $$KUBECONFIG_WAIT -gt 0 ] || exit ${BULID_EXIT_CODE_ON_NO_CLUSTER:-0}
@@ -94,12 +97,29 @@ services:
       set -e
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
-      kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack
+      kubectl get nodes
+
+      echo "Our ystack-proxy image should install ystack essentials ..."
+      kubectl get namespaces
+      kubectl-waitretry --for=condition=Ready -n ystack pod -l app=minio
+      kubectl-waitretry --for=condition=Ready -n ystack pod -l ystack-builds-registry=http
+      curl -f --retry 5 --retry-connrefused http://builds-registry.ystack.svc.cluster.local/v2/
 
       y-cluster-install-prometheus-operator
 
-      cd /usr/local/src/ystack/examples/basic-dev-inner-loop
-      y-skaffold run
+      kubectl create namespace monitoring
+      kubectl apply -k github.com/Yolean/kubernetes-assert/example-small?ref=f0318952dd4990bb3d4c9d163f8044f13e2b647d
+
+      kubectl-waitretry --for=condition=Ready -n monitoring pod --all
+
+      for EXAMPLE in $$EXAMPLES; do
+        echo "# Running example $$EXAMPLE ..."
+        cd /usr/local/src/ystack/examples/$$EXAMPLE
+        y-skaffold run
+      done
+
+      echo "# Running main specs ..."
+      y-assert
 
       if [ "$$KEEP_RUNNING" = "true" ]; then
         echo "Will stay running for manual work"

From e82469959b6de8f15aa65ab55f0bac87b716178b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 12:41:48 +0200
Subject: [PATCH 02/61] A gentle reminder

---
 docker-compose.test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index a0c2b06d..7a8ff47f 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -48,6 +48,7 @@ services:
     expose:
     - 8472
     - 10250
+    # It's a feature that we keep ystack with kubernetes-assert light enough to fit these limits
     mem_limit: 1500000000
     memswap_limit: 0
 

From 2d4ffbd0b867809e28c15c7658687827bc1f3980 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 12:42:14 +0200
Subject: [PATCH 03/61] Many projects will want to tweak build or deploy for CI
 runs

---
 bin/y-assert | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bin/y-assert b/bin/y-assert
index c7f79526..294b888c 100755
--- a/bin/y-assert
+++ b/bin/y-assert
@@ -32,6 +32,9 @@ case $ctx in
 esac
 skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|')
 
+[ -z "$PROFILES" ] && PROFILES="y-assert"
+skaffoldflags="$skaffoldflags -p $PROFILES"
+
 CIRUN=$1
 [ -z "$CIRUN" ] && CIRUN="run"
 

From 34b66e2017f79f0c4e0240b3e44a2c179b9d67fc Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 14:32:08 +0200
Subject: [PATCH 04/61] Oh, buildkitd had scale=0 because of a ./k3s
 kustomization

---
 README.md               | 1 +
 docker-compose.test.yml | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 4ac95705..ac4e3530 100644
--- a/README.md
+++ b/README.md
@@ -104,6 +104,7 @@ $compose down \
 sudo rm test/.kube/kubeconfig.yaml
 $compose up --build -d ystack-proxy
 export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
+kubectl -n ystack scale --replicas=1 statefulset buildkitd
 ```
 
 With kubectl access to the in-docker cluster you might want to start with
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 7a8ff47f..4319a2a5 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -104,7 +104,9 @@ services:
       kubectl get namespaces
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=minio
       kubectl-waitretry --for=condition=Ready -n ystack pod -l ystack-builds-registry=http
-      curl -f --retry 5 --retry-connrefused http://builds-registry.ystack.svc.cluster.local/v2/
+      kubectl -n ystack scale --replicas=1 statefulset buildkitd
+      kubectl-waitretry --for=condition=Ready -n ystack pod -l app=buildkitd
+      curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/
 
       y-cluster-install-prometheus-operator
 

From 9f6d6a6671bd412872147a04b7d73ad07f5796ba Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 14:32:58 +0200
Subject: [PATCH 05/61] ./test.sh now runs not only examples but also y-assert,
 which is

a milestone even if specs are just dummy
---
 docker-compose.test.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 4319a2a5..bd531ce2 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -79,9 +79,11 @@ services:
     - KUBECONFIG_WAIT=30
     - KEEP_RUNNING=false
     - EXAMPLES=sync-only in-cluster-build basic-dev-inner-loop
+    - CI=true
     volumes:
     - admin:/admin
     - ./examples:/usr/local/src/ystack/examples
+    - ./specs:/usr/local/src/ystack/specs
     entrypoint:
     - /bin/bash
     - -cx
@@ -109,8 +111,7 @@ services:
       curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/
 
       y-cluster-install-prometheus-operator
-
-      kubectl create namespace monitoring
+      kubectl get namespace monitoring 2>/dev/null || kubectl create namespace monitoring
       kubectl apply -k github.com/Yolean/kubernetes-assert/example-small?ref=f0318952dd4990bb3d4c9d163f8044f13e2b647d
 
       kubectl-waitretry --for=condition=Ready -n monitoring pod --all
@@ -122,7 +123,9 @@ services:
       done
 
       echo "# Running main specs ..."
-      y-assert
+      kubectl create namespace                         ystack-specs
+      kubectl config set-context --current --namespace=ystack-specs
+      (cd /usr/local/src/ystack/specs && y-assert)
 
       if [ "$$KEEP_RUNNING" = "true" ]; then
         echo "Will stay running for manual work"

From 7e2f839b8f80b650be09596dc49e0b5236e32823 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 14:33:56 +0200
Subject: [PATCH 06/61] Current operator, now that we depend on
 kubernetes-assert

---
 bin/y-cluster-install-prometheus-operator | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/y-cluster-install-prometheus-operator b/bin/y-cluster-install-prometheus-operator
index 3505dc68..d39bfbb8 100755
--- a/bin/y-cluster-install-prometheus-operator
+++ b/bin/y-cluster-install-prometheus-operator
@@ -2,6 +2,6 @@
 [ -z "$DEBUG" ] || set -x
 set -e
 
-OPERATOR_VERSION=d0a871b710de7b764c05ced98dbd1eb32a681790
+OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
 
 kubectl -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml

From 1b37e932b4fbf1fb11155a8bbfc3811abb89edc2 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 14:34:37 +0200
Subject: [PATCH 07/61] Cleanup

---
 .../apiserver-source-kafka.yaml               | 48 ----------
 .../broker-trigger-kafka.yaml                 | 23 -----
 example-knative-kafka/inspect-kafka001.yaml   | 50 ----------
 example-knative-kafka/kustomization.yaml      |  7 --
 example-knative-kafka/sequence-kafka.yaml     | 93 -------------------
 5 files changed, 221 deletions(-)
 delete mode 100644 example-knative-kafka/apiserver-source-kafka.yaml
 delete mode 100644 example-knative-kafka/broker-trigger-kafka.yaml
 delete mode 100644 example-knative-kafka/inspect-kafka001.yaml
 delete mode 100644 example-knative-kafka/kustomization.yaml
 delete mode 100644 example-knative-kafka/sequence-kafka.yaml

diff --git a/example-knative-kafka/apiserver-source-kafka.yaml b/example-knative-kafka/apiserver-source-kafka.yaml
deleted file mode 100644
index 36ffd787..00000000
--- a/example-knative-kafka/apiserver-source-kafka.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
----
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: events-sa
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: event-watcher
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - events
-    verbs:
-      - get
-      - list
-      - watch
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: k8s-ra-event-watcher
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: event-watcher
-subjects:
-  - kind: ServiceAccount
-    name: events-sa
-    # can  be reoved from rbac when we use Kustomize
-    namespace: test-knative-eventing
----
-apiVersion: sources.eventing.knative.dev/v1alpha1
-kind: ApiServerSource
-metadata:
-  name: testevents
-spec:
-  serviceAccountName: events-sa
-  mode: Resource
-  resources:
-    - apiVersion: v1
-      kind: Event
-  sink:
-    apiVersion: eventing.knative.dev/v1alpha1
-    kind: Broker
-    name: kafka001
diff --git a/example-knative-kafka/broker-trigger-kafka.yaml b/example-knative-kafka/broker-trigger-kafka.yaml
deleted file mode 100644
index d093f109..00000000
--- a/example-knative-kafka/broker-trigger-kafka.yaml
+++ /dev/null
@@ -1,23 +0,0 @@
-apiVersion: eventing.knative.dev/v1alpha1
-kind: Broker
-metadata:
-  name: kafka001
-spec:
-  channelTemplateSpec:
-    apiVersion: messaging.knative.dev/v1alpha1
-    kind: KafkaChannel
-    spec:
-      numPartitions: 1
-      replicationFactor: 1
----
-apiVersion: sources.eventing.knative.dev/v1alpha1
-kind: CronJobSource
-metadata:
-  name: every-second-minute-to-kafka001
-spec:
-  schedule: "*/2 * * * *"
-  data: '{"message": "Hello world!"}'
-  sink:
-    apiVersion: eventing.knative.dev/v1alpha1
-    kind: Broker
-    name: kafka001
diff --git a/example-knative-kafka/inspect-kafka001.yaml b/example-knative-kafka/inspect-kafka001.yaml
deleted file mode 100644
index aa585352..00000000
--- a/example-knative-kafka/inspect-kafka001.yaml
+++ /dev/null
@@ -1,50 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: inspect-kafka001
-  labels: &labels
-    app: inspect
-    category: eventing
-    type: broker
-    ref: kafka001
-spec:
-  replicas: 1
-  selector:
-    matchLabels: *labels
-  template:
-    metadata:
-      labels: *labels
-      annotations:
-        kafka-bootstrap: bootstrap.kafka:9092
-    spec:
-      containers:
-      - name: kafkacat
-        image: solsson/kafkacat:alpine@sha256:0d8cc772fea8399b17c6a62d7618a045ca5adc1b050eefdcc104a9c53b658b96
-        env:
-        - name: BOOTSTRAP
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.annotations['kafka-bootstrap']
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: TRIGGER_TO_INSPECT
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.labels['ref']
-        args:
-        - -b
-        - $(BOOTSTRAP)
-        - -C
-        - -t
-        - knative-messaging-kafka.$(NAMESPACE).$(TRIGGER_TO_INSPECT)-kn-trigger
-        - -f
-        - |-
-          \nKey (%K bytes): %k
-            Value (%S bytes): %s
-            Timestamp: %T
-            Partition: %p
-            Offset: %o
-            Headers: %h\n
-        - -u
diff --git a/example-knative-kafka/kustomization.yaml b/example-knative-kafka/kustomization.yaml
deleted file mode 100644
index 9a62ac31..00000000
--- a/example-knative-kafka/kustomization.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-namespace: test-knative-eventing
-resources:
-- sequence-kafka.yaml
-# The sequence is independent, the remainder belongs together
-- broker-trigger-kafka.yaml
-- apiserver-source-kafka.yaml
-- inspect-kafka001.yaml
diff --git a/example-knative-kafka/sequence-kafka.yaml b/example-knative-kafka/sequence-kafka.yaml
deleted file mode 100644
index 3002aa2a..00000000
--- a/example-knative-kafka/sequence-kafka.yaml
+++ /dev/null
@@ -1,93 +0,0 @@
-# https://github.com/knative/docs/blob/master/docs/eventing/samples/sequence/sequence-reply-to-event-display/README.md
----
-apiVersion: serving.knative.dev/v1beta1
-kind: Service
-metadata:
-  name: first
-spec:
-  template:
-    spec:
-      containers:
-        - image: us.gcr.io/probable-summer-223122/cmd-03315b715ae8f3e08e3a9378df706fbb@sha256:2656f39a7fcb6afd9fc79e7a4e215d14d651dc674f38020d1d18c6f04b220700
-          env:
-            - name: STEP
-              value: "0"
----
-apiVersion: serving.knative.dev/v1beta1
-kind: Service
-metadata:
-  name: second
-spec:
-  template:
-    spec:
-      containers:
-        - image: us.gcr.io/probable-summer-223122/cmd-03315b715ae8f3e08e3a9378df706fbb@sha256:2656f39a7fcb6afd9fc79e7a4e215d14d651dc674f38020d1d18c6f04b220700
-          env:
-            - name: STEP
-              value: "1"
----
-apiVersion: serving.knative.dev/v1beta1
-kind: Service
-metadata:
-  name: third
-spec:
-  template:
-    spec:
-      containers:
-        - image: us.gcr.io/probable-summer-223122/cmd-03315b715ae8f3e08e3a9378df706fbb@sha256:2656f39a7fcb6afd9fc79e7a4e215d14d651dc674f38020d1d18c6f04b220700
-          env:
-            - name: STEP
-              value: "2"
----
-apiVersion: messaging.knative.dev/v1alpha1
-kind: Sequence
-metadata:
-  name: sequence01
-spec:
-  # Broker uses spec
-  #channelTemplateSpec:
-  channelTemplate:
-    apiVersion: messaging.knative.dev/v1alpha1
-    kind: KafkaChannel
-    spec:
-      numPartitions: 1
-      replicationFactor: 1
-  steps:
-    - ref:
-        apiVersion: serving.knative.dev/v1alpha1
-        kind: Service
-        name: first
-    - ref:
-        apiVersion: serving.knative.dev/v1alpha1
-        kind: Service
-        name: second
-    - ref:
-        apiVersion: serving.knative.dev/v1alpha1
-        kind: Service
-        name: third
-  reply:
-    kind: Service
-    apiVersion: serving.knative.dev/v1alpha1
-    name: event-display
----
-apiVersion: serving.knative.dev/v1beta1
-kind: Service
-metadata:
-  name: event-display
-spec:
-  template:
-    spec:
-      containers:
-        - image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display
----
-apiVersion: sources.eventing.knative.dev/v1alpha1
-kind: CronJobSource
-metadata:
-  name: every-minute-to-sequence01
-spec:
-  schedule: "* * * * *"
-  data: '{"message": "Hello world!"}'
-  sink:
-    apiVersion: messaging.knative.dev/v1alpha1
-    kind: Sequence
-    name: sequence01

From af3ada33190a79f53fe4eeeb36ed5603eac2999a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 14:34:47 +0200
Subject: [PATCH 08/61] Cleanup

---
 runner.Dockerfile.dockerignore | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index cc62aead..563841ec 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -1,9 +1,10 @@
 *
+
+# ystack-runner components
 !bin/y-*
 bin/*-bin
 !bin/kubectl-*
 
-#!metrics-server/
 #!converge-generic/
 !minio/
 !registry/builds-service/
@@ -12,8 +13,6 @@ bin/*-bin
 !registry/node-update-hosts/
 !registry/generic,minio/
 !buildkit/
-#!monitoring/rbac-base/
-#!monitoring/instance-now/
 
 # k3s configuration
 !k3s/image/registries.yaml

From 6474e065c85eb10e23c87a19834d578fe4541a30 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 15:07:01 +0200
Subject: [PATCH 09/61] The actual specs, based on kubernetes-assert's example
 project

---
 specs/Dockerfile         |  9 +++++++++
 specs/kustomization.yaml | 10 ++++++++++
 specs/package-lock.json  |  5 +++++
 specs/package.json       |  4 ++++
 specs/skaffold.yaml      | 22 ++++++++++++++++++++++
 specs/src/dummy.spec.js  |  5 +++++
 6 files changed, 55 insertions(+)
 create mode 100644 specs/Dockerfile
 create mode 100644 specs/kustomization.yaml
 create mode 100644 specs/package-lock.json
 create mode 100644 specs/package.json
 create mode 100644 specs/skaffold.yaml
 create mode 100644 specs/src/dummy.spec.js

diff --git a/specs/Dockerfile b/specs/Dockerfile
new file mode 100644
index 00000000..11d27ae6
--- /dev/null
+++ b/specs/Dockerfile
@@ -0,0 +1,9 @@
+FROM yolean/assert-runtime-nodejs:d3630d89f1babc4a17f95e89a249030b8b501043@sha256:cb4ce7cddce3230420ae963c2ec5718b68254f98a8a00b6a0879d2aa672b9b56
+
+WORKDIR /usr/src/specs
+
+COPY --chown=nonroot:nogroup package*.json ./
+
+RUN npm ci --ignore-scripts
+
+COPY --chown=nonroot:nogroup . .
diff --git a/specs/kustomization.yaml b/specs/kustomization.yaml
new file mode 100644
index 00000000..ed2dbdf3
--- /dev/null
+++ b/specs/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+bases:
+- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=930ee2627ac53b7e356ce6a317226a010ffe3c8d
+namePrefix: ystack-
+commonLabels:
+  app: ystack
+images:
+- name: yolean/assert-runtime-nodejs
+  newName: builds-registry.ystack.svc.cluster.local/yolean/ystack-specs
diff --git a/specs/package-lock.json b/specs/package-lock.json
new file mode 100644
index 00000000..2f087b74
--- /dev/null
+++ b/specs/package-lock.json
@@ -0,0 +1,5 @@
+{
+  "name": "@yolean/ystack-specs",
+  "version": "1.0.0",
+  "lockfileVersion": 1
+}
diff --git a/specs/package.json b/specs/package.json
new file mode 100644
index 00000000..db2efec3
--- /dev/null
+++ b/specs/package.json
@@ -0,0 +1,4 @@
+{
+  "name": "@yolean/ystack-specs",
+  "version": "1.0.0"
+}
diff --git a/specs/skaffold.yaml b/specs/skaffold.yaml
new file mode 100644
index 00000000..69d9b00d
--- /dev/null
+++ b/specs/skaffold.yaml
@@ -0,0 +1,22 @@
+apiVersion: skaffold/v2beta4
+kind: Config
+metadata:
+  name: ystack-specs
+build:
+  tagPolicy:
+    gitCommit:
+      variant: CommitSha
+  artifacts:
+  - image: builds-registry.ystack.svc.cluster.local/yolean/ystack-specs
+    context: ./
+    custom:
+      buildCommand: y-build
+      dependencies:
+        dockerfile:
+          path: Dockerfile
+    sync:
+      infer: ["**/*"]
+deploy:
+  kustomize: {}
+profiles:
+- name: y-assert
diff --git a/specs/src/dummy.spec.js b/specs/src/dummy.spec.js
new file mode 100644
index 00000000..602eef9e
--- /dev/null
+++ b/specs/src/dummy.spec.js
@@ -0,0 +1,5 @@
+describe("Dummy spec", () => {
+  it("Just passes", () => {
+    expect(true).toBe(true);
+  });
+});

From 3b51d8a32c6bb2a5960f7e2982787af9c1d03c2d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 15:07:19 +0200
Subject: [PATCH 10/61] Now ./k3s scales both up and down :)

---
 README.md                             | 1 -
 docker-compose.test.yml               | 2 +-
 k3s/docker-ystack-proxy/entrypoint.sh | 2 ++
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index ac4e3530..4ac95705 100644
--- a/README.md
+++ b/README.md
@@ -104,7 +104,6 @@ $compose down \
 sudo rm test/.kube/kubeconfig.yaml
 $compose up --build -d ystack-proxy
 export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
-kubectl -n ystack scale --replicas=1 statefulset buildkitd
 ```
 
 With kubectl access to the in-docker cluster you might want to start with
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index bd531ce2..43e9c6c1 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -61,6 +61,7 @@ services:
       dockerfile: ./k3s/docker-ystack-proxy/Dockerfile
     environment:
     - KUBECONFIG_WAIT=30
+    - BUILDKITD_REPLICAS=1
     expose:
     - 80
     - 8547
@@ -106,7 +107,6 @@ services:
       kubectl get namespaces
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=minio
       kubectl-waitretry --for=condition=Ready -n ystack pod -l ystack-builds-registry=http
-      kubectl -n ystack scale --replicas=1 statefulset buildkitd
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=buildkitd
       curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/
 
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index 50ae28e7..0407d11b 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -13,6 +13,8 @@ kubectl-waitretry --for=condition=Ready node --all
 
 kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/
 
+[ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd
+
 NODE=agent
 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort})
 BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort})

From 9381eb7e0207e4a16af7506d6da0dff8403149e6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 15:41:04 +0200
Subject: [PATCH 11/61] kubie will by design not use a kubeconfig from the
 previous run

---
 README.md | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4ac95705..cf147885 100644
--- a/README.md
+++ b/README.md
@@ -101,9 +101,8 @@ docker volume rm ystack_admin 2> /dev/null || true
 compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
 $compose down \
   ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
-sudo rm test/.kube/kubeconfig.yaml
 $compose up --build -d ystack-proxy
-export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
+y-kubie ctx -f ./test/.kube/kubeconfig.yaml
 ```
 
 With kubectl access to the in-docker cluster you might want to start with

From 68ada7a99f5ae6f4ccd2cfcad7e5bf1987dcc800 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 20:51:53 +0200
Subject: [PATCH 12/61] Uses a skaffold build of the runtime from
 Yolean/kubernetes-assert#15

---
 specs/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/specs/Dockerfile b/specs/Dockerfile
index 11d27ae6..149db0ad 100644
--- a/specs/Dockerfile
+++ b/specs/Dockerfile
@@ -1,4 +1,4 @@
-FROM yolean/assert-runtime-nodejs:d3630d89f1babc4a17f95e89a249030b8b501043@sha256:cb4ce7cddce3230420ae963c2ec5718b68254f98a8a00b6a0879d2aa672b9b56
+FROM builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:a37053500670cc936b3b9fb05362bea5005ee20c
 
 WORKDIR /usr/src/specs
 

From 7a7523dc0da92af10a64eda5c333acc66793f7ad Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 20:57:47 +0200
Subject: [PATCH 13/61] Tries to avoid a sudo rm, but not get an old kubeconfig

---
 README.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cf147885..e6c7358a 100644
--- a/README.md
+++ b/README.md
@@ -101,8 +101,11 @@ docker volume rm ystack_admin 2> /dev/null || true
 compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
 $compose down \
   ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
+KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
+previous=$(stat -c %Y $KUBECONFIG 2>/dev/null)
 $compose up --build -d ystack-proxy
-y-kubie ctx -f ./test/.kube/kubeconfig.yaml
+while [ "$prev" = "$(stat -c %Y $KUBECONFIG 2>/dev/null)" ]; do echo "Waiting for $KUBECONFIG" && sleep 1; done
+y-kubie ctx -f $KUBECONFIG
 ```
 
 With kubectl access to the in-docker cluster you might want to start with

From b458d895980db7ee82730a4e35c63a7a219e5793 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 21:04:27 +0200
Subject: [PATCH 14/61] Starts a completion definition

---
 specs/src/assert-completion.spec.js | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 specs/src/assert-completion.spec.js

diff --git a/specs/src/assert-completion.spec.js b/specs/src/assert-completion.spec.js
new file mode 100644
index 00000000..7978198d
--- /dev/null
+++ b/specs/src/assert-completion.spec.js
@@ -0,0 +1,16 @@
+describe("Ystack specs completion", () => {
+
+  describe("Suite size", () => {
+
+    // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch
+    const ASSERT_FILES_MIN = 2;
+
+    it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => {
+      expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`))
+        .toBeGreaterThanOrEqual(ASSERT_FILES_MIN);
+    });
+
+  });
+
+  
+});

From 962c2324b62aa7148d2a06b20b9656ad4e7f831f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 5 Jun 2020 21:31:03 +0200
Subject: [PATCH 15/61] Fixes spec file count

---
 specs/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/specs/Dockerfile b/specs/Dockerfile
index 149db0ad..4c607d8b 100644
--- a/specs/Dockerfile
+++ b/specs/Dockerfile
@@ -1,4 +1,4 @@
-FROM builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:a37053500670cc936b3b9fb05362bea5005ee20c
+FROM builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:594d9886eca5175b90dcd96584578b8a42d5fc26
 
 WORKDIR /usr/src/specs
 

From 045820dcd7561bc7d133460133ead4ffdc40bcc4 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 6 Jun 2020 10:54:14 +0200
Subject: [PATCH 16/61] Fixed, but doesn't help, ystack-proxy gets old config

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e6c7358a..cfcc0a0c 100644
--- a/README.md
+++ b/README.md
@@ -104,7 +104,7 @@ $compose down \
 KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
 previous=$(stat -c %Y $KUBECONFIG 2>/dev/null)
 $compose up --build -d ystack-proxy
-while [ "$prev" = "$(stat -c %Y $KUBECONFIG 2>/dev/null)" ]; do echo "Waiting for $KUBECONFIG" && sleep 1; done
+while [ "$previous" = "$(stat -c %Y $KUBECONFIG 2>/dev/null)" ]; do echo "Waiting for $KUBECONFIG" && sleep 1; done
 y-kubie ctx -f $KUBECONFIG
 ```
 

From 0e0a89006293c062eb535b7ac29f882cc3316543 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 6 Jun 2020 11:14:05 +0200
Subject: [PATCH 17/61] Cleanup without sudo

---
 README.md                        |  5 +----
 {test => devcluster}/.gitignore  |  0
 docker-compose.dev-overrides.yml |  2 +-
 docker-compose.test.yml          | 13 +++++++++++++
 4 files changed, 15 insertions(+), 5 deletions(-)
 rename {test => devcluster}/.gitignore (100%)

diff --git a/README.md b/README.md
index cfcc0a0c..5d9f4805 100644
--- a/README.md
+++ b/README.md
@@ -101,11 +101,8 @@ docker volume rm ystack_admin 2> /dev/null || true
 compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
 $compose down \
   ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
-KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
-previous=$(stat -c %Y $KUBECONFIG 2>/dev/null)
 $compose up --build -d ystack-proxy
-while [ "$previous" = "$(stat -c %Y $KUBECONFIG 2>/dev/null)" ]; do echo "Waiting for $KUBECONFIG" && sleep 1; done
-y-kubie ctx -f $KUBECONFIG
+y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```
 
 With kubectl access to the in-docker cluster you might want to start with
diff --git a/test/.gitignore b/devcluster/.gitignore
similarity index 100%
rename from test/.gitignore
rename to devcluster/.gitignore
diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index 64d5a54a..2584c149 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -30,5 +30,5 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: $PWD/test
+      device: $PWD/devcluster
       o: bind
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 43e9c6c1..de2e0192 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -1,7 +1,20 @@
 version: '2.4'
 services:
 
+  cleanup:
+    image: busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209
+    command:
+    - sh
+    - -ce
+    - |
+      [ ! -f /admin/.kube/kubeconfig.yaml ] || rm -v /admin/.kube/kubeconfig.yaml
+      echo "Done. Exiting."
+    volumes:
+    - admin:/admin
+
   server:
+    depends_on:
+    - cleanup
     build:
       context: .
       dockerfile: ./k3s/image/Dockerfile

From 7ada5750c36eaae8c7b76983be09bf38e1eb35f3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 6 Jun 2020 11:14:15 +0200
Subject: [PATCH 18/61] I thought that node name resolvable from server would
 fix metric errors

E0606 09:05:46.810522       1 available_controller.go:420] v1beta1.metrics.k8s.io failed with: failing or missing response from https://10.43.103.169:443/apis/metrics.k8s.io/v1beta1: Get https://10.43.103.169:443/apis/metrics.k8s.io/v1beta1: net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)

... but no, they're still there.
---
 docker-compose.dev-overrides.yml      | 2 +-
 docker-compose.test.yml               | 5 +++--
 k3s/docker-ystack-proxy/entrypoint.sh | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index 2584c149..acce436e 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -5,7 +5,7 @@ services:
     ports:
     - 17143:17143
 
-  agent:
+  agent1:
     ports:
     # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead
     #- 80:80
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index de2e0192..25afc06a 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -45,7 +45,8 @@ services:
     mem_limit:  300000000
     memswap_limit: 0
 
-  agent:
+  agent1:
+    hostname: agent1
     depends_on:
     - server
     build:
@@ -68,7 +69,7 @@ services:
   ystack-proxy:
     depends_on:
     - server
-    - agent
+    - agent1
     build:
       context: .
       dockerfile: ./k3s/docker-ystack-proxy/Dockerfile
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index 0407d11b..e67a89e1 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -15,7 +15,7 @@ kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/
 
 [ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd
 
-NODE=agent
+NODE=agent1
 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort})
 BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort})
 

From df9d1c04ebb2a394f282ae7282a82fe0b2b17216 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 6 Jun 2020 11:17:11 +0200
Subject: [PATCH 19/61] See the first line of the script

---
 docker-compose.test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 25afc06a..a1dacf84 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -112,7 +112,7 @@ services:
         KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 ))
         echo "Waiting for a kubeconfig ..." && sleep 1
       done
-      set -e
+
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
       kubectl get nodes

From 65d94449ff4604fc4877e3add23d6badd516b961 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 6 Jun 2020 11:32:47 +0200
Subject: [PATCH 20/61] Cleanup would run at unwanted occations

---
 README.md               | 1 +
 docker-compose.test.yml | 2 --
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 5d9f4805..3e16e706 100644
--- a/README.md
+++ b/README.md
@@ -101,6 +101,7 @@ docker volume rm ystack_admin 2> /dev/null || true
 compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
 $compose down \
   ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
+$compose up cleanup
 $compose up --build -d ystack-proxy
 y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index a1dacf84..5c9cd306 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -13,8 +13,6 @@ services:
     - admin:/admin
 
   server:
-    depends_on:
-    - cleanup
     build:
       context: .
       dockerfile: ./k3s/image/Dockerfile

From 1d151fd72c987a8c2feaf6d4c784f5a3c3573c5e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 6 Jun 2020 11:38:46 +0200
Subject: [PATCH 21/61] Yay, the first release of crane has been published. Go
 local.

---
 bin/y-build-nothing |  9 +--------
 bin/y-crane         | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 8 deletions(-)
 create mode 100755 bin/y-crane

diff --git a/bin/y-build-nothing b/bin/y-build-nothing
index 2b87082e..94c48d85 100755
--- a/bin/y-build-nothing
+++ b/bin/y-build-nothing
@@ -24,13 +24,8 @@ case "$IMAGE" in
   * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
 esac
 
-[ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx"
-
-CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9
 NOTHING_IMAGE=busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209
 
-BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz")
-
 # TODO copy is significantly faster if the source image is already in the target repo
 
 # crane hangs for a long time if it doesn't know that the registry is plain http
@@ -38,6 +33,4 @@ NOTHING_IMAGE=$(echo $NOTHING_IMAGE | sed 's|.local|.local:80|')
 IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
 
 set -x
-kubectl $ctx -n ystack run --restart=Never --attach \
-  --image=$CRANE_IMAGE y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") -- \
-  cp $NOTHING_IMAGE $IMAGE
+y-crane cp $NOTHING_IMAGE $IMAGE
diff --git a/bin/y-crane b/bin/y-crane
new file mode 100755
index 00000000..6f538017
--- /dev/null
+++ b/bin/y-crane
@@ -0,0 +1,17 @@
+#!/bin/sh
+[ -z "$DEBUG" ] || set -x
+set -e
+YBIN="$(dirname $0)"
+
+version=0.1.0
+
+bin_name=crane \
+  bin_version=v${version} \
+  Darwin_url=https://github.com/google/go-containerregistry/releases/download/v${version}/go-containerregistry_${version}_Darwin_x86_64.tar.gz \
+  Darwin_sha256=4bef595a9d0b6781b452b302317f6b70c72e43cce664f817fa777ff72f9ffe5f \
+  Linux_url=https://github.com/google/go-containerregistry/releases/download/v${version}/go-containerregistry_${version}_Linux_x86_64.tar.gz \
+  Linux_sha256=ee5bb44cc9097835d12e6b34e1c1e3399efb8ed3dfb68ee0f28c6e66952dc335 \
+  bin_tgz_path=crane \
+  $YBIN/y-bin-dependency-download || exit $?
+
+$YBIN/y-crane-v${version}-bin "$@" || exit $?

From c057f8fe99190d5d50e90a3bf709e8c196822256 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 06:47:00 +0200
Subject: [PATCH 22/61] A concept for how to build without build

---
 examples/sync-to-runtime/.gitignore           |  0
 examples/sync-to-runtime/index.js             | 13 +++++++++++
 .../k8s/sync-to-runtime-deployment.yaml       | 21 +++++++++++++++++
 .../k8s/sync-to-runtime-service.yaml          |  9 ++++++++
 examples/sync-to-runtime/my-handler/index.js  |  3 +++
 examples/sync-to-runtime/skaffold.yaml        | 23 +++++++++++++++++++
 6 files changed, 69 insertions(+)
 create mode 100644 examples/sync-to-runtime/.gitignore
 create mode 100644 examples/sync-to-runtime/index.js
 create mode 100644 examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml
 create mode 100644 examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml
 create mode 100644 examples/sync-to-runtime/my-handler/index.js
 create mode 100644 examples/sync-to-runtime/skaffold.yaml

diff --git a/examples/sync-to-runtime/.gitignore b/examples/sync-to-runtime/.gitignore
new file mode 100644
index 00000000..e69de29b
diff --git a/examples/sync-to-runtime/index.js b/examples/sync-to-runtime/index.js
new file mode 100644
index 00000000..fa05f81b
--- /dev/null
+++ b/examples/sync-to-runtime/index.js
@@ -0,0 +1,13 @@
+// Unlike this example a typical runtime would contain the server and require the handler
+
+const handler = require('./my-handler');
+
+const http = require('http');
+const server = http.createServer((req, res) => {
+  handler().then(response => {
+    res.writeHead(200, { 'Content-Type': 'text/plain' });
+    res.end(response);
+  });
+});
+
+server.listen(8080, '0.0.0.0');
diff --git a/examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml b/examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml
new file mode 100644
index 00000000..5a7ea08d
--- /dev/null
+++ b/examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml
@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sync-to-runtime
+spec:
+  selector:
+    matchLabels:
+      app: ystack-examples-sync-to-runtime
+  template:
+    metadata:
+      labels:
+        app: ystack-examples-sync-to-runtime
+    spec:
+      containers:
+      - name: runtime
+        image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-to-runtime
+        ports:
+        - containerPort: 8080
+        command:
+        - node
+        - /usr/src/app
diff --git a/examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml b/examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml
new file mode 100644
index 00000000..32da6f07
--- /dev/null
+++ b/examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sync-to-runtime
+spec:
+  ports:
+  - port: 8080
+  selector:
+    app: ystack-examples-sync-to-runtime
diff --git a/examples/sync-to-runtime/my-handler/index.js b/examples/sync-to-runtime/my-handler/index.js
new file mode 100644
index 00000000..27393a35
--- /dev/null
+++ b/examples/sync-to-runtime/my-handler/index.js
@@ -0,0 +1,3 @@
+module.exports = async () => {
+  return "OK from handler 2";
+};
diff --git a/examples/sync-to-runtime/skaffold.yaml b/examples/sync-to-runtime/skaffold.yaml
new file mode 100644
index 00000000..6da62321
--- /dev/null
+++ b/examples/sync-to-runtime/skaffold.yaml
@@ -0,0 +1,23 @@
+apiVersion: skaffold/v2beta4
+kind: Config
+metadata:
+  name: sync-to-runtime
+build:
+  tagPolicy:
+    gitCommit:
+      variant: CommitSha
+  artifacts:
+  - image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-to-runtime
+    custom:
+      buildCommand: >-
+        y-build-like-sync
+        node:12.18.0-alpine3.11@sha256:53cd04d947b6c097fde5ea031622f917e8a61378890effd8d548b1204628a3fc
+    sync:
+      manual:
+      - src: '**'
+        dest: /usr/src/app
+deploy:
+  kubectl:
+    manifests:
+    - k8s/sync-to-runtime-service.yaml
+    - k8s/sync-to-runtime-deployment.yaml

From 40b821a5283ad529907868243cb7a0e836e32c4c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 07:01:02 +0200
Subject: [PATCH 23/61] Now the handler gets updated on regular skaffold sync

---
 examples/sync-to-runtime/index.js            | 7 +++++--
 examples/sync-to-runtime/my-handler/index.js | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/examples/sync-to-runtime/index.js b/examples/sync-to-runtime/index.js
index fa05f81b..7e7f01a3 100644
--- a/examples/sync-to-runtime/index.js
+++ b/examples/sync-to-runtime/index.js
@@ -1,10 +1,13 @@
 // Unlike this example a typical runtime would contain the server and require the handler
+// This runtime inefficiently re-requires the handler on every request
 
-const handler = require('./my-handler');
+// Must be the js file, not the folder, for require.cache deletion to work
+const HANDLER = './my-handler/index.js';
 
 const http = require('http');
 const server = http.createServer((req, res) => {
-  handler().then(response => {
+  delete require.cache[require.resolve(HANDLER)];
+  require(HANDLER)().then(response => {
     res.writeHead(200, { 'Content-Type': 'text/plain' });
     res.end(response);
   });
diff --git a/examples/sync-to-runtime/my-handler/index.js b/examples/sync-to-runtime/my-handler/index.js
index 27393a35..d46c56e3 100644
--- a/examples/sync-to-runtime/my-handler/index.js
+++ b/examples/sync-to-runtime/my-handler/index.js
@@ -1,3 +1,3 @@
 module.exports = async () => {
-  return "OK from handler 2";
+  return "OK from handler";
 };

From 96dd2521eae152f9d2ed31f4322918096249a5f0 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 07:12:15 +0200
Subject: [PATCH 24/61] ah, the script, s

---
 bin/y-build-like-sync | 57 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100755 bin/y-build-like-sync

diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync
new file mode 100755
index 00000000..1be77e84
--- /dev/null
+++ b/bin/y-build-like-sync
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+# Bash isn't a good tool for producing a tar from a build context
+# but this script is an experiment on how to compose an image
+# from a runtime and some static files using go-containerregistry's crane
+# The user shouldn't need to write Dockerfile or .dockerignore,
+# but we could probably use a generated Dockerfile + kaniko instead, or Buildpacks
+
+# Settings
+DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local
+[ -z "$BUILDS_REGISTRY" ] && BUILDS_REGISTRY=$DEFAULT_REGISTRY
+[ -z "$PUSH_REGISTRY" ]   && PUSH_REGISTRY=$DEFAULT_REGISTRY
+
+if [ "$(curl -s --connect-timeout 3 http://$BUILDS_REGISTRY/v2/)" != "{}" ]
+then
+  echo "ERROR Skaffold need local access to the builds registry for digest lookup"
+  echo "Registry: $BUILDS_REGISTRY"
+  echo "Look for y-stack's ingress or port-forward utilities"
+  exit 1
+fi
+
+[ -z "$IMAGE" ] && echo "No IMAGE env (from for example Skaffold)" && exit 1
+
+IMAGE=$IMAGE
+case "$IMAGE" in
+  $BUILDS_REGISTRY/* ) ;;
+  $PUSH_REGISTRY/* ) echo "Unlike y-build this script won't push to non-build registries" && exit 1 ;;
+  * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
+esac
+
+RUNTIME_IMAGE=$1
+[ -z "$RUNTIME_IMAGE" ] && echo "First argument must be a runtime image to append the layer to" \
+  && echo "To improve build times use a runtime image in the target repo" && exit 1
+
+# crane hangs for a long time if it doesn't know that the registry is plain http
+RUNTIME_IMAGE=$(echo $RUNTIME_IMAGE | sed 's|.local|.local:80|')
+IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
+
+# This is a PoC, let's make a lot of assumptions to simplify
+context=.
+src='**'
+dest=/usr/src/app
+# this avoids "tar: Removing leading `/' from member names" and could come in handy if we can't use --transform
+dest=usr/src/app
+
+list=$(mktemp)
+(cd $context; git --no-pager status --porcelain=v2 --short --untracked=all --no-renames -- . | cut -c 4-) > $list
+tar=$(mktemp)
+tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list
+rm $list
+
+set -x
+y-crane append --insecure -b $RUNTIME_IMAGE -f $tar -t $IMAGE
+set +x
+rm $tar

From 632c2245ff37138d450bfe512d73c12c7381c264 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 16:10:11 +0200
Subject: [PATCH 25/61] Includes unmodified files, honors .gitignore

---
 bin/y-build-like-sync               | 2 +-
 examples/sync-to-runtime/.gitignore | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 examples/sync-to-runtime/.gitignore

diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync
index 1be77e84..698f57a3 100755
--- a/bin/y-build-like-sync
+++ b/bin/y-build-like-sync
@@ -46,7 +46,7 @@ dest=/usr/src/app
 dest=usr/src/app
 
 list=$(mktemp)
-(cd $context; git --no-pager status --porcelain=v2 --short --untracked=all --no-renames -- . | cut -c 4-) > $list
+(cd $context; git ls-files -c -o --exclude-standard -- .) > $list
 tar=$(mktemp)
 tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list
 rm $list
diff --git a/examples/sync-to-runtime/.gitignore b/examples/sync-to-runtime/.gitignore
deleted file mode 100644
index e69de29b..00000000

From d6c0df70b84a299ead672d662e9903c87c7ebddb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 16:10:49 +0200
Subject: [PATCH 26/61] Gets dest path from skaffold, as it will vary widely

---
 bin/y-build-like-sync | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync
index 698f57a3..13c5fcd3 100755
--- a/bin/y-build-like-sync
+++ b/bin/y-build-like-sync
@@ -41,9 +41,11 @@ IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
 # This is a PoC, let's make a lot of assumptions to simplify
 context=.
 src='**'
-dest=/usr/src/app
+# assuming a single manual sync, which is the reasonable use case for a runtime
+[ ! -f skaffold.yaml ] && echo "This composition example assumes a sync defined in a skaffold.yaml" && exit 1
+dest=$(cat skaffold.yaml | grep 'dest:' | awk '{ print $2 }')
 # this avoids "tar: Removing leading `/' from member names" and could come in handy if we can't use --transform
-dest=usr/src/app
+dest=$(echo $dest | sed 's|^/||')
 
 list=$(mktemp)
 (cd $context; git ls-files -c -o --exclude-standard -- .) > $list

From bf39dacabb8b1e36df865b58d2ff7225aa46312f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 16:13:21 +0200
Subject: [PATCH 27/61] Adds recently introduced binaries to bin ignore

---
 bin/.gitignore | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bin/.gitignore b/bin/.gitignore
index cfb3f545..a3ee39fc 100644
--- a/bin/.gitignore
+++ b/bin/.gitignore
@@ -2,9 +2,14 @@
 
 # our executables are symlinked to real names
 buildctl
+container-structure-test
+crane
 helm
+kubie
+kustomize
 minikube
 mkcert
+promtool
 rio
 skaffold
 kubefwd

From 7f6336251868d8933f27de25874f0595660faa77 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 21:36:08 +0200
Subject: [PATCH 28/61] Prometheus proxy for kubefwd, and bundle more setup in
 k3s manifests

---
 README.md                                     |  3 ---
 bin/y-assert                                  |  4 +++
 docker-compose.test.yml                       |  9 ++-----
 .../kustomization.yaml                        |  1 +
 k3s/docker-ystack-proxy/Dockerfile            | 11 +++++++-
 k3s/docker-ystack-proxy/entrypoint.sh         |  2 +-
 k3s/docker-ystack-proxy/envoy.template.yaml   | 27 +++++++++++++++++++
 .../kustomization.yaml                        |  1 +
 runner.Dockerfile.dockerignore                |  2 ++
 9 files changed, 48 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 3e16e706..80d91152 100644
--- a/README.md
+++ b/README.md
@@ -105,6 +105,3 @@ $compose up cleanup
 $compose up --build -d ystack-proxy
 y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```
-
-With kubectl access to the in-docker cluster you might want to start with
-some of the steps from the "sut" script in [docker-compose.test.yml](./docker-compose.test.yml).
diff --git a/bin/y-assert b/bin/y-assert
index 294b888c..fc3719f9 100755
--- a/bin/y-assert
+++ b/bin/y-assert
@@ -58,6 +58,10 @@ EOF
 fi
 skaffoldflags="$skaffoldflags -n $namespace"
 
+# At this point we should know how to identify relevant assert metrics
+[ -z "$assertlabels" ] && assertlabels="namespace=\"$namespace\"";
+echo "$(log) Prometheus queries for test results will use labels: $assertlabels"
+
 set -x
 y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN
 
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 5c9cd306..49b91f40 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -77,6 +77,7 @@ services:
     expose:
     - 80
     - 8547
+    - 9090
     volumes:
     - admin:/admin
 
@@ -116,17 +117,11 @@ services:
       kubectl get nodes
 
       echo "Our ystack-proxy image should install ystack essentials ..."
-      kubectl get namespaces
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=minio
       kubectl-waitretry --for=condition=Ready -n ystack pod -l ystack-builds-registry=http
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=buildkitd
-      curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/
-
-      y-cluster-install-prometheus-operator
-      kubectl get namespace monitoring 2>/dev/null || kubectl create namespace monitoring
-      kubectl apply -k github.com/Yolean/kubernetes-assert/example-small?ref=f0318952dd4990bb3d4c9d163f8044f13e2b647d
-
       kubectl-waitretry --for=condition=Ready -n monitoring pod --all
+      curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/
 
       for EXAMPLE in $$EXAMPLES; do
         echo "# Running example $$EXAMPLE ..."
diff --git a/k3s/10-builds-registry-service/kustomization.yaml b/k3s/10-builds-registry-service/kustomization.yaml
index f23bbc00..3ceefe91 100644
--- a/k3s/10-builds-registry-service/kustomization.yaml
+++ b/k3s/10-builds-registry-service/kustomization.yaml
@@ -1,3 +1,4 @@
+namespace: ystack
 bases:
 - ../../registry/builds-service
 patchesStrategicMerge:
diff --git a/k3s/docker-ystack-proxy/Dockerfile b/k3s/docker-ystack-proxy/Dockerfile
index 25d0022b..1cdeb1a1 100644
--- a/k3s/docker-ystack-proxy/Dockerfile
+++ b/k3s/docker-ystack-proxy/Dockerfile
@@ -1,17 +1,26 @@
 FROM lachlanevenson/k8s-kubectl:v1.18.3@sha256:495f41912554fde10e41ac54e8df6fdbd0e859e893a4cab61c9224b296f3454d as kubectl
 
-RUN apk add --no-cache git
+RUN apk add --no-cache curl git
 
 COPY . /etc/ystack
+
 RUN set -ex; \
   mkdir -p /var/lib/rancher/k3s/server/manifests; \
   for base in \
       00-ystack-namespace \
       10-builds-registry-service \
+      20-monitoring-namespace \
+      25-kubernetes-assert \
+      27-y-assert-support \
       registry,minio,buildkit; do \
     kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \
   done
 
+RUN set -ex; \
+  eval $(cat /etc/ystack/bin/y-cluster-install-prometheus-operator | grep OPERATOR_VERSION=); \
+  curl -sLS -o /var/lib/rancher/k3s/server/manifests/ystack-00-prometheus-operator.yaml \
+    https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml;
+
 FROM envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89
 
 COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index e67a89e1..a766b024 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -11,7 +11,7 @@ set -e
 cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
 kubectl-waitretry --for=condition=Ready node --all
 
-kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/
+kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 
 [ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd
 
diff --git a/k3s/docker-ystack-proxy/envoy.template.yaml b/k3s/docker-ystack-proxy/envoy.template.yaml
index c34e7c8b..c56f9c2b 100644
--- a/k3s/docker-ystack-proxy/envoy.template.yaml
+++ b/k3s/docker-ystack-proxy/envoy.template.yaml
@@ -33,6 +33,19 @@ static_resources:
           "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           stat_prefix: tcp_stats
           cluster: "buildkitd"
+  - name: prometheus
+    address:
+      socket_address:
+        protocol: TCP
+        address: 0.0.0.0
+        port_value: 9090
+    filter_chains:
+    - filters:
+      - name: tcp
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+          stat_prefix: tcp_stats
+          cluster: "prometheus"
   clusters:
   - name: registry
     type: STRICT_DNS
@@ -62,3 +75,17 @@ static_resources:
               socket_address:
                 address: {{ node }}
                 port_value: {{ buildkit_nodeport }}
+  - name: prometheus
+    type: STRICT_DNS
+    connect_timeout: 30s
+    dns_lookup_family: V4_ONLY
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: service_prometheus
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: {{ node }}
+                port_value: {{ prometheus_nodeport }}
diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/registry,minio,buildkit/kustomization.yaml
index 4e47c13c..e26d53a3 100644
--- a/k3s/registry,minio,buildkit/kustomization.yaml
+++ b/k3s/registry,minio,buildkit/kustomization.yaml
@@ -1,3 +1,4 @@
+namespace: ystack
 bases:
 - ../../buildkit
 - ../../minio/standalone,defaultsecret
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index 563841ec..40e98679 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -13,11 +13,13 @@ bin/*-bin
 !registry/node-update-hosts/
 !registry/generic,minio/
 !buildkit/
+!y-assert-support/
 
 # k3s configuration
 !k3s/image/registries.yaml
 !k3s/00-ystack-namespace
 !k3s/10-builds-registry-service
 !k3s/registry,minio,buildkit
+!k3s/2*
 !k3s/docker-ystack-proxy/envoy.template.yaml
 !k3s/docker-ystack-proxy/entrypoint.sh

From 1ef8864563d7bf0592a86dc2c28791a39872c004 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 7 Jun 2020 21:54:54 +0200
Subject: [PATCH 29/61] Once again prometheus-operator defies attempts to apply
 together

with other resources
---
 docker-compose.dev-overrides.yml      | 4 +++-
 docker-compose.test.yml               | 2 +-
 k3s/docker-ystack-proxy/Dockerfile    | 8 ++------
 k3s/docker-ystack-proxy/entrypoint.sh | 6 ++++++
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index acce436e..96667156 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -18,8 +18,10 @@ services:
     ports:
     # To run skaffold build/run/dev locally
     - 80:80
-    # To support y-build as skaffold build (TODO - proxying fails)
+    # To support y-build as skaffold build
     - 8547:8547
+    # To support y-assert
+    - 9090:9090
 
   sut:
     environment:
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 49b91f40..d11b637f 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -92,7 +92,7 @@ services:
     environment:
     - KUBECONFIG_WAIT=30
     - KEEP_RUNNING=false
-    - EXAMPLES=sync-only in-cluster-build basic-dev-inner-loop
+    - EXAMPLES=sync-only sync-to-runtime in-cluster-build basic-dev-inner-loop
     - CI=true
     volumes:
     - admin:/admin
diff --git a/k3s/docker-ystack-proxy/Dockerfile b/k3s/docker-ystack-proxy/Dockerfile
index 1cdeb1a1..50637a05 100644
--- a/k3s/docker-ystack-proxy/Dockerfile
+++ b/k3s/docker-ystack-proxy/Dockerfile
@@ -1,6 +1,6 @@
 FROM lachlanevenson/k8s-kubectl:v1.18.3@sha256:495f41912554fde10e41ac54e8df6fdbd0e859e893a4cab61c9224b296f3454d as kubectl
 
-RUN apk add --no-cache curl git
+RUN apk add --no-cache git
 
 COPY . /etc/ystack
 
@@ -16,16 +16,12 @@ RUN set -ex; \
     kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \
   done
 
-RUN set -ex; \
-  eval $(cat /etc/ystack/bin/y-cluster-install-prometheus-operator | grep OPERATOR_VERSION=); \
-  curl -sLS -o /var/lib/rancher/k3s/server/manifests/ystack-00-prometheus-operator.yaml \
-    https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml;
-
 FROM envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89
 
 COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl
 
 COPY bin/kubectl-waitretry /usr/local/bin/kubectl-waitretry
+COPY bin/y-cluster-install-prometheus-operator /usr/local/bin/y-cluster-install-prometheus-operator
 
 COPY --from=kubectl /var/lib/rancher/k3s/server/manifests /var/lib/rancher/k3s/server/manifests
 
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index a766b024..4e1ca700 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -11,6 +11,10 @@ set -e
 cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
 kubectl-waitretry --for=condition=Ready node --all
 
+# TODO this script does apply but a second apply is really slow on resource strapped clusters, should be avoided if we want this script to be idempotent
+y-cluster-install-prometheus-operator
+sleep 10 # TODO Can we explicitly wait for CRDs to be registered?
+
 kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 
 [ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd
@@ -18,11 +22,13 @@ kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 NODE=agent1
 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort})
 BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort})
+PROMETHEUS=$(kubectl -n ystack get service prometheus-nodeport -o jsonpath={.spec.ports[0].nodePort})
 
 cat envoy.template.yaml \
   | sed "s|{{ node }}|$NODE|g" \
   | sed "s|{{ registry_nodeport }}|$REGISTRY|g" \
   | sed "s|{{ buildkit_nodeport }}|$BUILDKIT|g" \
+  | sed "s|{{ prometheus_nodeport }}|$PROMETHEUS|g" \
   > /envoy.yaml
 
 # TODO do we pass on signals?

From bc4df4963c884ed6ad63eda596a598a29b22a716 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 06:44:39 +0200
Subject: [PATCH 30/61] The new in-docker resources

---
 k3s/20-monitoring-namespace/kustomization.yaml       |  2 ++
 .../monitoring-namespace.yaml                        |  4 ++++
 k3s/25-kubernetes-assert/kustomization.yaml          |  3 +++
 k3s/27-y-assert-support/kustomization.yaml           |  5 +++++
 .../prometheus-nodeport-service.yaml                 | 12 ++++++++++++
 5 files changed, 26 insertions(+)
 create mode 100644 k3s/20-monitoring-namespace/kustomization.yaml
 create mode 100644 k3s/20-monitoring-namespace/monitoring-namespace.yaml
 create mode 100644 k3s/25-kubernetes-assert/kustomization.yaml
 create mode 100644 k3s/27-y-assert-support/kustomization.yaml
 create mode 100644 k3s/27-y-assert-support/prometheus-nodeport-service.yaml

diff --git a/k3s/20-monitoring-namespace/kustomization.yaml b/k3s/20-monitoring-namespace/kustomization.yaml
new file mode 100644
index 00000000..b3775d93
--- /dev/null
+++ b/k3s/20-monitoring-namespace/kustomization.yaml
@@ -0,0 +1,2 @@
+resources:
+- monitoring-namespace.yaml
diff --git a/k3s/20-monitoring-namespace/monitoring-namespace.yaml b/k3s/20-monitoring-namespace/monitoring-namespace.yaml
new file mode 100644
index 00000000..d3252360
--- /dev/null
+++ b/k3s/20-monitoring-namespace/monitoring-namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: monitoring
diff --git a/k3s/25-kubernetes-assert/kustomization.yaml b/k3s/25-kubernetes-assert/kustomization.yaml
new file mode 100644
index 00000000..3394cc8a
--- /dev/null
+++ b/k3s/25-kubernetes-assert/kustomization.yaml
@@ -0,0 +1,3 @@
+namespace: monitoring
+bases:
+- github.com/Yolean/kubernetes-assert/example-small?ref=f0318952dd4990bb3d4c9d163f8044f13e2b647d
diff --git a/k3s/27-y-assert-support/kustomization.yaml b/k3s/27-y-assert-support/kustomization.yaml
new file mode 100644
index 00000000..1471162a
--- /dev/null
+++ b/k3s/27-y-assert-support/kustomization.yaml
@@ -0,0 +1,5 @@
+namespace: ystack
+bases:
+- ../../y-assert-support
+resources:
+- prometheus-nodeport-service.yaml
diff --git a/k3s/27-y-assert-support/prometheus-nodeport-service.yaml b/k3s/27-y-assert-support/prometheus-nodeport-service.yaml
new file mode 100644
index 00000000..d727f4a1
--- /dev/null
+++ b/k3s/27-y-assert-support/prometheus-nodeport-service.yaml
@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus-nodeport
+spec:
+  type: NodePort
+  ports:
+  - port: 9090
+    protocol: TCP
+  selector:
+    app: buildkitd
+    status: accepts-new-builds

From 181b0fca4451f7ee0b8d5f6561cd01903349020b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 06:45:46 +0200
Subject: [PATCH 31/61] The Prometheus proxy resource

---
 y-assert-support/kustomization.yaml           |  3 +
 .../prometheus-proxy-deployment.yaml          | 98 +++++++++++++++++++
 y-assert-support/prometheus-service.yaml      | 11 +++
 3 files changed, 112 insertions(+)
 create mode 100644 y-assert-support/kustomization.yaml
 create mode 100644 y-assert-support/prometheus-proxy-deployment.yaml
 create mode 100644 y-assert-support/prometheus-service.yaml

diff --git a/y-assert-support/kustomization.yaml b/y-assert-support/kustomization.yaml
new file mode 100644
index 00000000..4d24d1e8
--- /dev/null
+++ b/y-assert-support/kustomization.yaml
@@ -0,0 +1,3 @@
+resources:
+- prometheus-service.yaml
+- prometheus-proxy-deployment.yaml
diff --git a/y-assert-support/prometheus-proxy-deployment.yaml b/y-assert-support/prometheus-proxy-deployment.yaml
new file mode 100644
index 00000000..13b03347
--- /dev/null
+++ b/y-assert-support/prometheus-proxy-deployment.yaml
@@ -0,0 +1,98 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: prometheus-proxy
+  labels:
+    app: prometheus-proxy
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: prometheus-proxy
+  template:
+    metadata:
+      labels:
+        app: prometheus-proxy
+    spec:
+      containers:
+      - name: proxy
+        image: envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89
+        ports:
+        - containerPort: 9090
+        command:
+        - /usr/local/bin/envoy
+        - -l
+        -   info
+        - --config-yaml
+        - |
+          admin:
+            access_log_path: /dev/null
+            address:
+              socket_address:
+                protocol: TCP
+                address: 0.0.0.0
+                port_value: 9901
+          static_resources:
+            listeners:
+            - name: listener_0
+              address:
+                socket_address:
+                  protocol: TCP
+                  address: 0.0.0.0
+                  port_value: 9090
+              filter_chains:
+              - filters:
+                - name: envoy.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                    stat_prefix: prometheus-proxy
+                    access_log:
+                    - name: envoy.file_access_log
+                      filter:
+                        not_health_check_filter:  {}
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog
+                        path: /dev/stdout
+                        typed_json_format:
+                          start_time: "%START_TIME%"
+                          req_method: "%REQ(:METHOD)%"
+                          req_path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+                          resp_code: "%RESPONSE_CODE%"
+                          resp_flags: "%RESPONSE_FLAGS%"
+                          bytes_recv: "%BYTES_RECEIVED%"
+                          bytes_sent: "%BYTES_SENT%"
+                          duration: "%DURATION%"
+                          agent: "%REQ(USER-AGENT)%"
+                          req_id: "%REQ(X-REQUEST-ID)%"
+                          upstream_host: "%UPSTREAM_HOST%"
+                          resp_upstream_service_time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"
+                          resp_redirect: "%RESP(LOCATION)%"
+                          req_content_type: "%REQ(CONTENT-TYPE)%"
+                    route_config:
+                      name: prometheus
+                      virtual_hosts:
+                      - name: prometheus
+                        domains: ["*"]
+                        routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: prometheus
+                            timeout: 30s
+                    http_filters:
+                    - name: envoy.filters.http.router
+            clusters:
+            - name: prometheus
+              type: STRICT_DNS
+              connect_timeout: 0.25s
+              lb_policy: ROUND_ROBIN
+              load_assignment:
+                cluster_name: prometheus
+                endpoints:
+                - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          protocol: TCP
+                          port_value: 9090
+                          address: prometheus-now.monitoring.svc.cluster.local
diff --git a/y-assert-support/prometheus-service.yaml b/y-assert-support/prometheus-service.yaml
new file mode 100644
index 00000000..f5bd5127
--- /dev/null
+++ b/y-assert-support/prometheus-service.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: prometheus
+spec:
+  selector:
+    app: prometheus-proxy
+  ports:
+    - protocol: TCP
+      port: 9090
+      targetPort: 9090

From 86b0949531eb04c48116aafa1a6f46f239efd91b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 07:56:55 +0200
Subject: [PATCH 32/61] Since we're still depending on a script for the
 operator,

why not install all of kubernetes-assert there?
---
 ...ter-install-prometheus-operator => y-cluster-assert-install} | 0
 bin/y-cluster-provision-gke                                     | 2 +-
 bin/y-cluster-provision-k3s-multipass                           | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename bin/{y-cluster-install-prometheus-operator => y-cluster-assert-install} (100%)

diff --git a/bin/y-cluster-install-prometheus-operator b/bin/y-cluster-assert-install
similarity index 100%
rename from bin/y-cluster-install-prometheus-operator
rename to bin/y-cluster-assert-install
diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 30fae328..253e87e9 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -124,4 +124,4 @@ kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \
   echo "Warning: failed to create builds registry storage bucket"
 
-y-cluster-install-prometheus-operator
+y-cluster-assert-install
diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass
index 7c9d3af6..78e8d904 100755
--- a/bin/y-cluster-provision-k3s-multipass
+++ b/bin/y-cluster-provision-k3s-multipass
@@ -30,7 +30,7 @@ KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack
 
 #KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server
 
-KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator
+KUBECONFIG="$KUBECONFIG.tmp" y-cluster-assert-install
 
 y-kubeconfig-import "$KUBECONFIG.tmp"
 

From fab8608ab759035e94bc058588d4fefb6570d10c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 09:17:22 +0200
Subject: [PATCH 33/61] Improves reliability of test provision, by a bit of
 dependency ordering

---
 bin/y-cluster-assert-install                   | 18 +++++++++++++++++-
 bin/y-cluster-provision-gke                    | 13 +++++++------
 bin/y-cluster-provision-k3s-multipass          |  4 ++--
 docker-compose.test.yml                        |  4 +++-
 k3s/10-minio/kustomization.yaml                |  3 +++
 .../builds-registry-magic-numbers.yaml         |  0
 .../builds-registry-replicas-1.yaml            |  0
 .../kustomization.yaml                         |  2 ++
 k3s/20-monitoring-namespace/kustomization.yaml |  2 --
 .../monitoring-namespace.yaml                  |  4 ----
 k3s/25-kubernetes-assert/kustomization.yaml    |  3 ---
 .../kustomization.yaml                         |  0
 .../prometheus-nodeport-service.yaml           |  0
 .../buildkitd-nodeport-service.yaml            |  0
 .../buildkitd-replicas-0.yaml                  |  0
 .../kustomization.yaml                         |  3 ---
 k3s/docker-ystack-proxy/Dockerfile             | 11 +++++------
 k3s/docker-ystack-proxy/entrypoint.sh          |  7 ++++---
 runner.Dockerfile.dockerignore                 | 12 +++++++++---
 19 files changed, 52 insertions(+), 34 deletions(-)
 create mode 100644 k3s/10-minio/kustomization.yaml
 rename k3s/{10-builds-registry-service => 20-builds-registry}/builds-registry-magic-numbers.yaml (100%)
 rename k3s/{registry,minio,buildkit => 20-builds-registry}/builds-registry-replicas-1.yaml (100%)
 rename k3s/{10-builds-registry-service => 20-builds-registry}/kustomization.yaml (64%)
 delete mode 100644 k3s/20-monitoring-namespace/kustomization.yaml
 delete mode 100644 k3s/20-monitoring-namespace/monitoring-namespace.yaml
 delete mode 100644 k3s/25-kubernetes-assert/kustomization.yaml
 rename k3s/{27-y-assert-support => 30-y-assert-support}/kustomization.yaml (100%)
 rename k3s/{27-y-assert-support => 30-y-assert-support}/prometheus-nodeport-service.yaml (100%)
 rename k3s/{registry,minio,buildkit => 40-buildkit}/buildkitd-nodeport-service.yaml (100%)
 rename k3s/{registry,minio,buildkit => 40-buildkit}/buildkitd-replicas-0.yaml (100%)
 rename k3s/{registry,minio,buildkit => 40-buildkit}/kustomization.yaml (51%)

diff --git a/bin/y-cluster-assert-install b/bin/y-cluster-assert-install
index d39bfbb8..e6b45949 100755
--- a/bin/y-cluster-assert-install
+++ b/bin/y-cluster-assert-install
@@ -3,5 +3,21 @@
 set -e
 
 OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
+KUBERNETES_ASSERT_VERSION=f0318952dd4990bb3d4c9d163f8044f13e2b647d
 
-kubectl -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml
+ctx=$1
+case $ctx in
+  "--context="*) shift 1 ;;
+  *) echo "Initial arg must be --context=" && exit 1 ;;
+esac
+
+# If we're to make this script idempotent we must make sure that re-applying the operator's bundle isn't painfully slow
+kubectl $ctx create namespace monitoring
+
+kubectl $ctx -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml
+
+kubectl-waitretry $ctx -n default --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-operator
+
+kubectl $ctx -n monitoring apply -k github.com/Yolean/kubernetes-assert/example-small?ref=$KUBERNETES_ASSERT_VERSION
+
+kubectl-waitretry $ctx -n monitoring --for=condition=Ready pod --all
diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke
index 253e87e9..a262487f 100755
--- a/bin/y-cluster-provision-gke
+++ b/bin/y-cluster-provision-gke
@@ -94,19 +94,20 @@ esac
 
 gcloud container clusters create "$NAME" $FLAGS "$@"
 
-GKE_CONTEXT_NAME=$(kubectl config current-context)
+GKE_CONTEXT_NAME=$(kubectl $ctx config current-context)
+ctx="--context=${GKE_CONTEXT_NAME}"
 
 echo "For outbound Internet access: y-cluster-provision-gke-network-outbound-enable"
 
-kubectl create namespace ystack
+kubectl $ctx create namespace ystack
 
-kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
+kubectl $ctx -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/
 
 ### To be moved to registry provision script, but we do want the initial context name
 
 [ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds
 
-kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
+kubectl $ctx create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME}
 
 gcloud compute backend-buckets list
 gsutil ls
@@ -120,8 +121,8 @@ metadata:
 spec: {}
 EOF
 
-kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \
+kubectl $ctx apply -f gke/registry/builds-gcs-storagebucket.yaml || \
   gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \
   echo "Warning: failed to create builds registry storage bucket"
 
-y-cluster-assert-install
+y-cluster-assert-install $ctx
diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass
index 78e8d904..c64aa37a 100755
--- a/bin/y-cluster-provision-k3s-multipass
+++ b/bin/y-cluster-provision-k3s-multipass
@@ -26,11 +26,11 @@ multipass exec "$VM_NAME" -- sudo cat /etc/rancher/k3s/k3s.yaml \
 
 KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local
 
-KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack
+KUBECONFIG="$KUBECONFIG.tmp" kubectl --context=local create namespace ystack
 
 #KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server
 
-KUBECONFIG="$KUBECONFIG.tmp" y-cluster-assert-install
+KUBECONFIG="$KUBECONFIG.tmp" y-cluster-assert-install --context=local
 
 y-kubeconfig-import "$KUBECONFIG.tmp"
 
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index d11b637f..7df573e7 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -73,6 +73,7 @@ services:
       dockerfile: ./k3s/docker-ystack-proxy/Dockerfile
     environment:
     - KUBECONFIG_WAIT=30
+    # Scaled to zero by default because we'll want to introduce automated scale to demand
     - BUILDKITD_REPLICAS=1
     expose:
     - 80
@@ -120,9 +121,10 @@ services:
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=minio
       kubectl-waitretry --for=condition=Ready -n ystack pod -l ystack-builds-registry=http
       kubectl-waitretry --for=condition=Ready -n ystack pod -l app=buildkitd
-      kubectl-waitretry --for=condition=Ready -n monitoring pod --all
       curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/
 
+      y-cluster-assert-install --context=default
+
       for EXAMPLE in $$EXAMPLES; do
         echo "# Running example $$EXAMPLE ..."
         cd /usr/local/src/ystack/examples/$$EXAMPLE
diff --git a/k3s/10-minio/kustomization.yaml b/k3s/10-minio/kustomization.yaml
new file mode 100644
index 00000000..a19d3220
--- /dev/null
+++ b/k3s/10-minio/kustomization.yaml
@@ -0,0 +1,3 @@
+namespace: ystack
+bases:
+- ../../minio/standalone,defaultsecret
diff --git a/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml b/k3s/20-builds-registry/builds-registry-magic-numbers.yaml
similarity index 100%
rename from k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml
rename to k3s/20-builds-registry/builds-registry-magic-numbers.yaml
diff --git a/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml b/k3s/20-builds-registry/builds-registry-replicas-1.yaml
similarity index 100%
rename from k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml
rename to k3s/20-builds-registry/builds-registry-replicas-1.yaml
diff --git a/k3s/10-builds-registry-service/kustomization.yaml b/k3s/20-builds-registry/kustomization.yaml
similarity index 64%
rename from k3s/10-builds-registry-service/kustomization.yaml
rename to k3s/20-builds-registry/kustomization.yaml
index 3ceefe91..1f72dce7 100644
--- a/k3s/10-builds-registry-service/kustomization.yaml
+++ b/k3s/20-builds-registry/kustomization.yaml
@@ -1,5 +1,7 @@
 namespace: ystack
 bases:
 - ../../registry/builds-service
+- ../../registry/generic,minio
 patchesStrategicMerge:
 - builds-registry-magic-numbers.yaml
+- builds-registry-replicas-1.yaml
diff --git a/k3s/20-monitoring-namespace/kustomization.yaml b/k3s/20-monitoring-namespace/kustomization.yaml
deleted file mode 100644
index b3775d93..00000000
--- a/k3s/20-monitoring-namespace/kustomization.yaml
+++ /dev/null
@@ -1,2 +0,0 @@
-resources:
-- monitoring-namespace.yaml
diff --git a/k3s/20-monitoring-namespace/monitoring-namespace.yaml b/k3s/20-monitoring-namespace/monitoring-namespace.yaml
deleted file mode 100644
index d3252360..00000000
--- a/k3s/20-monitoring-namespace/monitoring-namespace.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: monitoring
diff --git a/k3s/25-kubernetes-assert/kustomization.yaml b/k3s/25-kubernetes-assert/kustomization.yaml
deleted file mode 100644
index 3394cc8a..00000000
--- a/k3s/25-kubernetes-assert/kustomization.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-namespace: monitoring
-bases:
-- github.com/Yolean/kubernetes-assert/example-small?ref=f0318952dd4990bb3d4c9d163f8044f13e2b647d
diff --git a/k3s/27-y-assert-support/kustomization.yaml b/k3s/30-y-assert-support/kustomization.yaml
similarity index 100%
rename from k3s/27-y-assert-support/kustomization.yaml
rename to k3s/30-y-assert-support/kustomization.yaml
diff --git a/k3s/27-y-assert-support/prometheus-nodeport-service.yaml b/k3s/30-y-assert-support/prometheus-nodeport-service.yaml
similarity index 100%
rename from k3s/27-y-assert-support/prometheus-nodeport-service.yaml
rename to k3s/30-y-assert-support/prometheus-nodeport-service.yaml
diff --git a/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml b/k3s/40-buildkit/buildkitd-nodeport-service.yaml
similarity index 100%
rename from k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml
rename to k3s/40-buildkit/buildkitd-nodeport-service.yaml
diff --git a/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml b/k3s/40-buildkit/buildkitd-replicas-0.yaml
similarity index 100%
rename from k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml
rename to k3s/40-buildkit/buildkitd-replicas-0.yaml
diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/40-buildkit/kustomization.yaml
similarity index 51%
rename from k3s/registry,minio,buildkit/kustomization.yaml
rename to k3s/40-buildkit/kustomization.yaml
index e26d53a3..5d13bf1b 100644
--- a/k3s/registry,minio,buildkit/kustomization.yaml
+++ b/k3s/40-buildkit/kustomization.yaml
@@ -2,10 +2,7 @@ namespace: ystack
 bases:
 - ../../buildkit
 - ../../minio/standalone,defaultsecret
-- ../../registry/generic,minio
 resources:
 - buildkitd-nodeport-service.yaml
 patchesStrategicMerge:
-- builds-registry-replicas-1.yaml
-# Making it work with ystack-proxy is a todo, and maybe we could do some kind of scale up on demand
 - buildkitd-replicas-0.yaml
diff --git a/k3s/docker-ystack-proxy/Dockerfile b/k3s/docker-ystack-proxy/Dockerfile
index 50637a05..252800ff 100644
--- a/k3s/docker-ystack-proxy/Dockerfile
+++ b/k3s/docker-ystack-proxy/Dockerfile
@@ -4,15 +4,15 @@ RUN apk add --no-cache git
 
 COPY . /etc/ystack
 
+# Install the things we need to proxy, but nothing more because big apply chunks are hard to maintain
 RUN set -ex; \
   mkdir -p /var/lib/rancher/k3s/server/manifests; \
   for base in \
       00-ystack-namespace \
-      10-builds-registry-service \
-      20-monitoring-namespace \
-      25-kubernetes-assert \
-      27-y-assert-support \
-      registry,minio,buildkit; do \
+      10-minio \
+      20-builds-registry \
+      30-y-assert-support \
+      40-buildkit; do \
     kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \
   done
 
@@ -21,7 +21,6 @@ FROM envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411
 COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl
 
 COPY bin/kubectl-waitretry /usr/local/bin/kubectl-waitretry
-COPY bin/y-cluster-install-prometheus-operator /usr/local/bin/y-cluster-install-prometheus-operator
 
 COPY --from=kubectl /var/lib/rancher/k3s/server/manifests /var/lib/rancher/k3s/server/manifests
 
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index 4e1ca700..501093a7 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -11,9 +11,10 @@ set -e
 cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
 kubectl-waitretry --for=condition=Ready node --all
 
-# TODO this script does apply but a second apply is really slow on resource strapped clusters, should be avoided if we want this script to be idempotent
-y-cluster-install-prometheus-operator
-sleep 10 # TODO Can we explicitly wait for CRDs to be registered?
+# Might speed up provision, due to the dependency minio -> registry -> builds, but should't be necessary
+kubectl apply -f /var/lib/rancher/k3s/server/manifests/ystack-00-ystack-namespace.yaml
+kubectl apply -f /var/lib/rancher/k3s/server/manifests/ystack-10-minio.yaml
+kubectl-waitretry -n ystack --for=condition=Ready pod minio-0
 
 kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index 40e98679..25c6218f 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -17,9 +17,15 @@ bin/*-bin
 
 # k3s configuration
 !k3s/image/registries.yaml
-!k3s/00-ystack-namespace
-!k3s/10-builds-registry-service
-!k3s/registry,minio,buildkit
+!k3s/0*
+!k3s/1*
 !k3s/2*
+!k3s/3*
+!k3s/4*
+!k3s/5*
+!k3s/6*
+!k3s/7*
+!k3s/8*
+!k3s/9*
 !k3s/docker-ystack-proxy/envoy.template.yaml
 !k3s/docker-ystack-proxy/entrypoint.sh

From 3a8acf70d770d1008460b93c7ad9b5ff397a6828 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 11:12:04 +0200
Subject: [PATCH 34/61] Proxy both prometheus and alertmanager, for y-assert
 and rules development

---
 k3s/30-y-assert-support/kustomization.yaml    |  2 +-
 .../monitoring-nodeport-service.yaml          | 17 +++++
 .../prometheus-nodeport-service.yaml          | 12 ---
 k3s/docker-ystack-proxy/entrypoint.sh         |  5 +-
 k3s/docker-ystack-proxy/envoy.template.yaml   | 27 +++++++
 y-assert-support/kustomization.yaml           |  4 +-
 ....yaml => monitoring-proxy-deployment.yaml} | 74 +++++++++++++++++--
 y-assert-support/monitoring-service.yaml      | 16 ++++
 y-assert-support/prometheus-service.yaml      | 11 ---
 9 files changed, 135 insertions(+), 33 deletions(-)
 create mode 100644 k3s/30-y-assert-support/monitoring-nodeport-service.yaml
 delete mode 100644 k3s/30-y-assert-support/prometheus-nodeport-service.yaml
 rename y-assert-support/{prometheus-proxy-deployment.yaml => monitoring-proxy-deployment.yaml} (53%)
 create mode 100644 y-assert-support/monitoring-service.yaml
 delete mode 100644 y-assert-support/prometheus-service.yaml

diff --git a/k3s/30-y-assert-support/kustomization.yaml b/k3s/30-y-assert-support/kustomization.yaml
index 1471162a..aa648dd9 100644
--- a/k3s/30-y-assert-support/kustomization.yaml
+++ b/k3s/30-y-assert-support/kustomization.yaml
@@ -2,4 +2,4 @@ namespace: ystack
 bases:
 - ../../y-assert-support
 resources:
-- prometheus-nodeport-service.yaml
+- monitoring-nodeport-service.yaml
diff --git a/k3s/30-y-assert-support/monitoring-nodeport-service.yaml b/k3s/30-y-assert-support/monitoring-nodeport-service.yaml
new file mode 100644
index 00000000..cfd89975
--- /dev/null
+++ b/k3s/30-y-assert-support/monitoring-nodeport-service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: monitoring-nodeport
+spec:
+  type: NodePort
+  selector:
+    app: monitoring-proxy
+  ports:
+  - name: prometheus
+    port: 9090
+    protocol: TCP
+    targetPort: 9090
+  - name: alertmanager
+    port: 9093
+    protocol: TCP
+    targetPort: 9093
diff --git a/k3s/30-y-assert-support/prometheus-nodeport-service.yaml b/k3s/30-y-assert-support/prometheus-nodeport-service.yaml
deleted file mode 100644
index d727f4a1..00000000
--- a/k3s/30-y-assert-support/prometheus-nodeport-service.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: prometheus-nodeport
-spec:
-  type: NodePort
-  ports:
-  - port: 9090
-    protocol: TCP
-  selector:
-    app: buildkitd
-    status: accepts-new-builds
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index 501093a7..8b337337 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -23,13 +23,16 @@ kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 NODE=agent1
 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort})
 BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort})
-PROMETHEUS=$(kubectl -n ystack get service prometheus-nodeport -o jsonpath={.spec.ports[0].nodePort})
+# Assuming ordering is predictable ...
+PROMETHEUS=$(kubectl -n ystack get service monitoring-nodeport -o jsonpath={.spec.ports[0].nodePort})
+ALERTMANAGER=$(kubectl -n ystack get service monitoring-nodeport -o jsonpath={.spec.ports[1].nodePort})
 
 cat envoy.template.yaml \
   | sed "s|{{ node }}|$NODE|g" \
   | sed "s|{{ registry_nodeport }}|$REGISTRY|g" \
   | sed "s|{{ buildkit_nodeport }}|$BUILDKIT|g" \
   | sed "s|{{ prometheus_nodeport }}|$PROMETHEUS|g" \
+  | sed "s|{{ alertmanager_nodeport }}|$ALERTMANAGER|g" \
   > /envoy.yaml
 
 # TODO do we pass on signals?
diff --git a/k3s/docker-ystack-proxy/envoy.template.yaml b/k3s/docker-ystack-proxy/envoy.template.yaml
index c56f9c2b..4bf7045e 100644
--- a/k3s/docker-ystack-proxy/envoy.template.yaml
+++ b/k3s/docker-ystack-proxy/envoy.template.yaml
@@ -46,6 +46,19 @@ static_resources:
           "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
           stat_prefix: tcp_stats
           cluster: "prometheus"
+  - name: alertmanager
+    address:
+      socket_address:
+        protocol: TCP
+        address: 0.0.0.0
+        port_value: 9093
+    filter_chains:
+    - filters:
+      - name: tcp
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+          stat_prefix: tcp_stats
+          cluster: "alertmanager"
   clusters:
   - name: registry
     type: STRICT_DNS
@@ -89,3 +102,17 @@ static_resources:
               socket_address:
                 address: {{ node }}
                 port_value: {{ prometheus_nodeport }}
+  - name: alertmanager
+    type: STRICT_DNS
+    connect_timeout: 30s
+    dns_lookup_family: V4_ONLY
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: service_alertmanager
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: {{ node }}
+                port_value: {{ alertmanager_nodeport }}
diff --git a/y-assert-support/kustomization.yaml b/y-assert-support/kustomization.yaml
index 4d24d1e8..47b3d055 100644
--- a/y-assert-support/kustomization.yaml
+++ b/y-assert-support/kustomization.yaml
@@ -1,3 +1,3 @@
 resources:
-- prometheus-service.yaml
-- prometheus-proxy-deployment.yaml
+- monitoring-service.yaml
+- monitoring-proxy-deployment.yaml
diff --git a/y-assert-support/prometheus-proxy-deployment.yaml b/y-assert-support/monitoring-proxy-deployment.yaml
similarity index 53%
rename from y-assert-support/prometheus-proxy-deployment.yaml
rename to y-assert-support/monitoring-proxy-deployment.yaml
index 13b03347..988ee59e 100644
--- a/y-assert-support/prometheus-proxy-deployment.yaml
+++ b/y-assert-support/monitoring-proxy-deployment.yaml
@@ -1,24 +1,25 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: prometheus-proxy
+  name: monitoring-proxy
   labels:
-    app: prometheus-proxy
+    app: monitoring-proxy
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: prometheus-proxy
+      app: monitoring-proxy
   template:
     metadata:
       labels:
-        app: prometheus-proxy
+        app: monitoring-proxy
     spec:
       containers:
       - name: proxy
         image: envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89
         ports:
         - containerPort: 9090
+        - containerPort: 9093
         command:
         - /usr/local/bin/envoy
         - -l
@@ -34,7 +35,7 @@ spec:
                 port_value: 9901
           static_resources:
             listeners:
-            - name: listener_0
+            - name: prometheus
               address:
                 socket_address:
                   protocol: TCP
@@ -45,7 +46,7 @@ spec:
                 - name: envoy.http_connection_manager
                   typed_config:
                     "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
-                    stat_prefix: prometheus-proxy
+                    stat_prefix: prometheus
                     access_log:
                     - name: envoy.file_access_log
                       filter:
@@ -81,6 +82,53 @@ spec:
                             timeout: 30s
                     http_filters:
                     - name: envoy.filters.http.router
+            - name: alertmanager
+              address:
+                socket_address:
+                  protocol: TCP
+                  address: 0.0.0.0
+                  port_value: 9093
+              filter_chains:
+              - filters:
+                - name: envoy.http_connection_manager
+                  typed_config:
+                    "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager
+                    stat_prefix: alertmanager
+                    access_log:
+                    - name: envoy.file_access_log
+                      filter:
+                        not_health_check_filter:  {}
+                      typed_config:
+                        "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog
+                        path: /dev/stdout
+                        typed_json_format:
+                          start_time: "%START_TIME%"
+                          req_method: "%REQ(:METHOD)%"
+                          req_path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%"
+                          resp_code: "%RESPONSE_CODE%"
+                          resp_flags: "%RESPONSE_FLAGS%"
+                          bytes_recv: "%BYTES_RECEIVED%"
+                          bytes_sent: "%BYTES_SENT%"
+                          duration: "%DURATION%"
+                          agent: "%REQ(USER-AGENT)%"
+                          req_id: "%REQ(X-REQUEST-ID)%"
+                          upstream_host: "%UPSTREAM_HOST%"
+                          resp_upstream_service_time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%"
+                          resp_redirect: "%RESP(LOCATION)%"
+                          req_content_type: "%REQ(CONTENT-TYPE)%"
+                    route_config:
+                      name: alertmanager
+                      virtual_hosts:
+                      - name: alertmanager
+                        domains: ["*"]
+                        routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: alertmanager
+                            timeout: 30s
+                    http_filters:
+                    - name: envoy.filters.http.router
             clusters:
             - name: prometheus
               type: STRICT_DNS
@@ -96,3 +144,17 @@ spec:
                           protocol: TCP
                           port_value: 9090
                           address: prometheus-now.monitoring.svc.cluster.local
+            - name: alertmanager
+              type: STRICT_DNS
+              connect_timeout: 0.25s
+              lb_policy: ROUND_ROBIN
+              load_assignment:
+                cluster_name: alertmanager
+                endpoints:
+                - lb_endpoints:
+                  - endpoint:
+                      address:
+                        socket_address:
+                          protocol: TCP
+                          port_value: 9090
+                          address: alertmanager-main.monitoring.svc.cluster.local
diff --git a/y-assert-support/monitoring-service.yaml b/y-assert-support/monitoring-service.yaml
new file mode 100644
index 00000000..6e38a6c4
--- /dev/null
+++ b/y-assert-support/monitoring-service.yaml
@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: monitoring
+spec:
+  selector:
+    app: monitoring-proxy
+  ports:
+  - name: prometheus
+    port: 9090
+    protocol: TCP
+    targetPort: 9090
+  - name: alertmanager
+    port: 9093
+    protocol: TCP
+    targetPort: 9093
diff --git a/y-assert-support/prometheus-service.yaml b/y-assert-support/prometheus-service.yaml
deleted file mode 100644
index f5bd5127..00000000
--- a/y-assert-support/prometheus-service.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: prometheus
-spec:
-  selector:
-    app: prometheus-proxy
-  ports:
-    - protocol: TCP
-      port: 9090
-      targetPort: 9090

From 62ce52b82225a00e42ba35336da5c8bdd1004a34 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 11:13:34 +0200
Subject: [PATCH 35/61] Workaround for: failed to get instance ID from cloud
 provider,

with the symptom that all pods stay in pending state with an error
saying that nodes have an "uninitialized" taint.
Can't find any reason why this error started to occur now.
---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 80d91152..08462bd0 100644
--- a/README.md
+++ b/README.md
@@ -102,6 +102,8 @@ compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrid
 $compose down \
   ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
 $compose up cleanup
+$compose up --build -d server
+$compose up --build -d agent1
 $compose up --build -d ystack-proxy
 y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```

From d452a87e348152697b07f9a27816e770898d5107 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 11:41:34 +0200
Subject: [PATCH 36/61] Fixes and documents the monitoring proxy required for
 local y-assert

---
 README.md                                      | 18 ++++++++++++++++++
 docker-compose.dev-overrides.yml               |  1 +
 docker-compose.test.yml                        |  1 +
 .../monitoring-proxy-deployment.yaml           |  2 +-
 4 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 08462bd0..a31aafb5 100644
--- a/README.md
+++ b/README.md
@@ -107,3 +107,21 @@ $compose up --build -d agent1
 $compose up --build -d ystack-proxy
 y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```
+
+To add monitoring support run `y-cluster-assert-install`.
+
+For [dev loops](./examples/) and `y-assert` the docker stack replaces `y-kubefwd` (hard to use in CI) with container ports:
+
+You need `cat /etc/hosts | grep 127.0.0 | grep cluster.local` to have something like:
+```
+127.0.0.1	builds-registry.ystack.svc.cluster.local
+127.0.0.1	buildkitd.ystack.svc.cluster.local
+127.0.0.1	monitoring.ystack.svc.cluster.local
+```
+
+Test using:
+```
+curl http://builds-registry.ystack.svc.cluster.local/v2/
+curl http://monitoring.ystack.svc.cluster.local:9090/api/v1/alertmanagers | jq '.data.activeAlertmanagers[0]'
+curl http://monitoring.ystack.svc.cluster.local:9093/api/v2/status
+```
diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index 96667156..43952af3 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -22,6 +22,7 @@ services:
     - 8547:8547
     # To support y-assert
     - 9090:9090
+    - 9093:9093
 
   sut:
     environment:
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 7df573e7..5f74ec25 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -79,6 +79,7 @@ services:
     - 80
     - 8547
     - 9090
+    - 9093
     volumes:
     - admin:/admin
 
diff --git a/y-assert-support/monitoring-proxy-deployment.yaml b/y-assert-support/monitoring-proxy-deployment.yaml
index 988ee59e..2b4a3d6b 100644
--- a/y-assert-support/monitoring-proxy-deployment.yaml
+++ b/y-assert-support/monitoring-proxy-deployment.yaml
@@ -156,5 +156,5 @@ spec:
                       address:
                         socket_address:
                           protocol: TCP
-                          port_value: 9090
+                          port_value: 9093
                           address: alertmanager-main.monitoring.svc.cluster.local

From 99cd4faec145e082fe364438f7e20c7a33f95268 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 11:55:14 +0200
Subject: [PATCH 37/61] Faster specs build and less boilerplate using runtime

---
 README.md                                 |  6 ++++--
 specs/Dockerfile                          |  9 ---------
 specs/{src => }/assert-completion.spec.js |  0
 specs/{src => }/dummy.spec.js             |  0
 specs/package-lock.json                   |  5 -----
 specs/package.json                        |  4 ----
 specs/skaffold.yaml                       | 11 ++++++-----
 7 files changed, 10 insertions(+), 25 deletions(-)
 delete mode 100644 specs/Dockerfile
 rename specs/{src => }/assert-completion.spec.js (100%)
 rename specs/{src => }/dummy.spec.js (100%)
 delete mode 100644 specs/package-lock.json
 delete mode 100644 specs/package.json

diff --git a/README.md b/README.md
index a31aafb5..add57080 100644
--- a/README.md
+++ b/README.md
@@ -110,8 +110,8 @@ y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 
 To add monitoring support run `y-cluster-assert-install`.
 
-For [dev loops](./examples/) and `y-assert` the docker stack replaces `y-kubefwd` (hard to use in CI) with container ports:
-
+For [dev loops](./examples/) and `y-assert` the docker stack replaces `y-kubefwd` (hard to use in CI)
+with container ports.
 You need `cat /etc/hosts | grep 127.0.0 | grep cluster.local` to have something like:
 ```
 127.0.0.1	builds-registry.ystack.svc.cluster.local
@@ -125,3 +125,5 @@ curl http://builds-registry.ystack.svc.cluster.local/v2/
 curl http://monitoring.ystack.svc.cluster.local:9090/api/v1/alertmanagers | jq '.data.activeAlertmanagers[0]'
 curl http://monitoring.ystack.svc.cluster.local:9093/api/v2/status
 ```
+
+Start a dev loop for actual asserts using `cd specs; y-skaffold --cache-artifacts=false dev` and start editing specs/*.spec.js.
diff --git a/specs/Dockerfile b/specs/Dockerfile
deleted file mode 100644
index 4c607d8b..00000000
--- a/specs/Dockerfile
+++ /dev/null
@@ -1,9 +0,0 @@
-FROM builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:594d9886eca5175b90dcd96584578b8a42d5fc26
-
-WORKDIR /usr/src/specs
-
-COPY --chown=nonroot:nogroup package*.json ./
-
-RUN npm ci --ignore-scripts
-
-COPY --chown=nonroot:nogroup . .
diff --git a/specs/src/assert-completion.spec.js b/specs/assert-completion.spec.js
similarity index 100%
rename from specs/src/assert-completion.spec.js
rename to specs/assert-completion.spec.js
diff --git a/specs/src/dummy.spec.js b/specs/dummy.spec.js
similarity index 100%
rename from specs/src/dummy.spec.js
rename to specs/dummy.spec.js
diff --git a/specs/package-lock.json b/specs/package-lock.json
deleted file mode 100644
index 2f087b74..00000000
--- a/specs/package-lock.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-  "name": "@yolean/ystack-specs",
-  "version": "1.0.0",
-  "lockfileVersion": 1
-}
diff --git a/specs/package.json b/specs/package.json
deleted file mode 100644
index db2efec3..00000000
--- a/specs/package.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-  "name": "@yolean/ystack-specs",
-  "version": "1.0.0"
-}
diff --git a/specs/skaffold.yaml b/specs/skaffold.yaml
index 69d9b00d..0835205c 100644
--- a/specs/skaffold.yaml
+++ b/specs/skaffold.yaml
@@ -10,12 +10,13 @@ build:
   - image: builds-registry.ystack.svc.cluster.local/yolean/ystack-specs
     context: ./
     custom:
-      buildCommand: y-build
-      dependencies:
-        dockerfile:
-          path: Dockerfile
+      buildCommand: >-
+        y-build-like-sync
+        builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:825e4973566d3a736e3bbb689c533a33a3092173
     sync:
-      infer: ["**/*"]
+      manual:
+      - src: '**'
+        dest: /usr/src/specs
 deploy:
   kustomize: {}
 profiles:

From 835da10a73ea340e41e4e96a262cea5ce67525dd Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 12:05:40 +0200
Subject: [PATCH 38/61] Finally asserting stuff, but still with skaffold dev

---
 .editorconfig                   |  7 +++++++
 specs/assert-assert.spec.js     | 11 +++++++++++
 specs/assert-completion.spec.js | 16 ----------------
 specs/minio.spec.js             | 12 ++++++++++++
 4 files changed, 30 insertions(+), 16 deletions(-)
 create mode 100644 .editorconfig
 create mode 100644 specs/assert-assert.spec.js
 delete mode 100644 specs/assert-completion.spec.js
 create mode 100644 specs/minio.spec.js

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 00000000..fa3df9b6
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,7 @@
+root = true
+
+[*]
+charset = utf-8
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2
diff --git a/specs/assert-assert.spec.js b/specs/assert-assert.spec.js
new file mode 100644
index 00000000..4e0219ee
--- /dev/null
+++ b/specs/assert-assert.spec.js
@@ -0,0 +1,11 @@
+// This is quite imporant to keep up-to-date while we're getting used to non-interactive Jest watch
+const ASSERT_FILES_MIN = 3;
+
+describe("Ystack specs validity", () => {
+
+  it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => {
+    expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`))
+      .toBeGreaterThanOrEqual(ASSERT_FILES_MIN);
+  });
+  
+});
diff --git a/specs/assert-completion.spec.js b/specs/assert-completion.spec.js
deleted file mode 100644
index 7978198d..00000000
--- a/specs/assert-completion.spec.js
+++ /dev/null
@@ -1,16 +0,0 @@
-describe("Ystack specs completion", () => {
-
-  describe("Suite size", () => {
-
-    // This is quite imporant to keep up-to-date while we're learning non-interactive jest watch
-    const ASSERT_FILES_MIN = 2;
-
-    it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => {
-      expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`))
-        .toBeGreaterThanOrEqual(ASSERT_FILES_MIN);
-    });
-
-  });
-
-  
-});
diff --git a/specs/minio.spec.js b/specs/minio.spec.js
new file mode 100644
index 00000000..71659531
--- /dev/null
+++ b/specs/minio.spec.js
@@ -0,0 +1,12 @@
+describe("Ystack minio", () => {
+  
+  describe("http", () => {
+    
+    it("The blobs-minio service (a legacy name) uses port 80", async () => {
+      const blobs = await fetch('http://blobs-minio.ystack/').then(res => res.text());
+      expect(blobs).toMatch(/AccessDenied/);  
+    });
+    
+  });
+
+});

From 2cb8a7d45c9a532373b17199da4e8080fdcd7b32 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 12:08:10 +0200
Subject: [PATCH 39/61] Still very basic specs

---
 specs/assert-assert.spec.js | 2 +-
 specs/registry.spec.js      | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 specs/registry.spec.js

diff --git a/specs/assert-assert.spec.js b/specs/assert-assert.spec.js
index 4e0219ee..9f872a09 100644
--- a/specs/assert-assert.spec.js
+++ b/specs/assert-assert.spec.js
@@ -1,5 +1,5 @@
 // This is quite imporant to keep up-to-date while we're getting used to non-interactive Jest watch
-const ASSERT_FILES_MIN = 3;
+const ASSERT_FILES_MIN = 4;
 
 describe("Ystack specs validity", () => {
 
diff --git a/specs/registry.spec.js b/specs/registry.spec.js
new file mode 100644
index 00000000..c63c358d
--- /dev/null
+++ b/specs/registry.spec.js
@@ -0,0 +1,8 @@
+describe("Builds registry", () => {
+      
+  it("Registry is up, with storage", async () => {
+    const v2 = await fetch('http://builds-registry.ystack/v2/').then(res => res.json());
+    expect(v2).toEqual({});
+  });
+  
+});

From 843a7fc7416e349bea573fb43538283d34ae4878 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 12:14:47 +0200
Subject: [PATCH 40/61] Covers the essential http endpoints in ystack

---
 specs/dummy.spec.js      |  5 -----
 specs/monitoring.spec.js | 17 +++++++++++++++++
 2 files changed, 17 insertions(+), 5 deletions(-)
 delete mode 100644 specs/dummy.spec.js
 create mode 100644 specs/monitoring.spec.js

diff --git a/specs/dummy.spec.js b/specs/dummy.spec.js
deleted file mode 100644
index 602eef9e..00000000
--- a/specs/dummy.spec.js
+++ /dev/null
@@ -1,5 +0,0 @@
-describe("Dummy spec", () => {
-  it("Just passes", () => {
-    expect(true).toBe(true);
-  });
-});
diff --git a/specs/monitoring.spec.js b/specs/monitoring.spec.js
new file mode 100644
index 00000000..b05e752f
--- /dev/null
+++ b/specs/monitoring.spec.js
@@ -0,0 +1,17 @@
+describe("Monitoring", () => {
+
+  describe("Ystack's proxy", () => {
+
+    it("Has Prometheus on port 9090", async () => {
+      const prom = await fetch('http://monitoring.ystack:9090/api/v1/status/runtimeinfo').then(res => res.json());
+      expect(prom).toHaveProperty('data.reloadConfigSuccess', true);
+    });
+
+    it("Has Alertmanager on port 9093", async () => {
+      const alerts = await fetch('http://monitoring.ystack:9093/api/v2/alerts').then(res => res.json());
+      expect(alerts).toBeInstanceOf(Array);
+    });
+
+  });
+
+});

From 5d1a75ff490498dc8acc138bfe1f12ebac5068b2 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 15:22:31 +0200
Subject: [PATCH 41/61] Stay running until there's zero test failures

---
 bin/y-assert | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/bin/y-assert b/bin/y-assert
index fc3719f9..24263f6e 100755
--- a/bin/y-assert
+++ b/bin/y-assert
@@ -60,9 +60,17 @@ skaffoldflags="$skaffoldflags -n $namespace"
 
 # At this point we should know how to identify relevant assert metrics
 [ -z "$assertlabels" ] && assertlabels="namespace=\"$namespace\"";
-echo "$(log) Prometheus queries for test results will use labels: $assertlabels"
 
-set -x
 y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN
 
+echo "$(log) Prometheus queries for test results will use labels: $assertlabels"
 echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: */=/') -o name | xargs kubectl $ctx delete"
+
+query=assertions_failed{$assertlabels}
+
+[ -z "$INTERVAL" ] && INTERVAL=5
+# This loop will continuously list errors and TODO! must fail on http errors or absence of the metric
+until curl -f -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \
+  | jq -r '.data.result[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end'
+do sleep $INTERVAL
+done

From c36b8973a0f690b521766dc22565d685c1cedc70 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 15:38:18 +0200
Subject: [PATCH 42/61] Adds a jq criteria to pass (i.e. exit) only if the
 expected metric had values

---
 bin/y-assert | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/bin/y-assert b/bin/y-assert
index 24263f6e..9ef2642f 100755
--- a/bin/y-assert
+++ b/bin/y-assert
@@ -69,8 +69,10 @@ echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: *
 query=assertions_failed{$assertlabels}
 
 [ -z "$INTERVAL" ] && INTERVAL=5
-# This loop will continuously list errors and TODO! must fail on http errors or absence of the metric
-until curl -f -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \
-  | jq -r '.data.result[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end'
+# This loop will continuously list errors and must fail on http errors or absence of the metric
+until curl -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \
+  | jq -r '.data.result | if length > 0 then . else error("No results") end | .[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end'
 do sleep $INTERVAL
 done
+
+echo "Zero failures now with: curl -s --data query=$query $MONITORING_HOST:9090/api/v1/query"

From 0d4b993502cea0419985257bab3a9bb14b578162 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 8 Jun 2020 16:27:32 +0200
Subject: [PATCH 43/61] I think this is good enough for a PoC

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index add57080..2dcd5772 100644
--- a/README.md
+++ b/README.md
@@ -126,4 +126,4 @@ curl http://monitoring.ystack.svc.cluster.local:9090/api/v1/alertmanagers | jq '
 curl http://monitoring.ystack.svc.cluster.local:9093/api/v2/status
 ```
 
-Start a dev loop for actual asserts using `cd specs; y-skaffold --cache-artifacts=false dev` and start editing specs/*.spec.js.
+Start a dev loop for actual asserts using `cd specs; y-skaffold --cache-artifacts=false dev` and start editing specs/*.spec.js. Run `y-assert` for CI-like runs until completion.

From 3514d6d2e7fe27de99d73252796ef81238eb1b1a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 10 Jun 2020 11:03:38 +0200
Subject: [PATCH 44/61] Verifies monitoring endpoints early in y-assert

---
 bin/y-assert | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bin/y-assert b/bin/y-assert
index 9ef2642f..486cbdf9 100755
--- a/bin/y-assert
+++ b/bin/y-assert
@@ -22,6 +22,11 @@ using ephemeral namespaces.
 # What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor
 TTL="janitor/ttl: 23h"
 
+[ -z "$MONITORING_HOST" ] && MONITORING_HOST=http://monitoring.ystack.svc.cluster.local
+
+! curl -f -s --connect-timeout 3 $MONITORING_HOST:9090/ >/dev/null && echo "Failed to access the Prometheus endpoint" && exit 1
+! curl -f -s --connect-timeout 3 $MONITORING_HOST:9093/ >/dev/null && echo "Failed to access the Alertmanager endpoint" && exit 1
+
 ctx=$1
 case $ctx in
   "--context="*) shift 1 ;;

From a5b926382ef8f8002fd6b34d33ef72eb0b08e137 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 10 Jun 2020 11:04:19 +0200
Subject: [PATCH 45/61] Runs same crane build that developers get locally

---
 runner.Dockerfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/runner.Dockerfile b/runner.Dockerfile
index 1cf5741c..f00393f3 100644
--- a/runner.Dockerfile
+++ b/runner.Dockerfile
@@ -36,13 +36,13 @@ RUN y-buildctl
 COPY bin/y-container-structure-test /usr/local/src/ystack/bin/
 RUN y-container-structure-test
 
+COPY bin/y-crane /usr/local/src/ystack/bin/
+RUN y-crane
+
 ENV SKAFFOLD_UPDATE_CHECK=false
 COPY bin/y-skaffold /usr/local/src/ystack/bin/
 RUN y-skaffold
 
-COPY --from=gcr.io/go-containerregistry/crane:57766c1fee7bdc67c4e70680efffed535d50b459 \
-  /ko-app/crane /usr/local/bin/crane
-
 COPY . /usr/local/src/ystack
 WORKDIR /usr/local/src/ystack
 

From e105d113ec0f5370291374359e56fadc46fd1e8f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 10 Jun 2020 11:05:03 +0200
Subject: [PATCH 46/61] Only add port number if it isn't there already

---
 bin/y-build-like-sync | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync
index 13c5fcd3..a19c46f9 100755
--- a/bin/y-build-like-sync
+++ b/bin/y-build-like-sync
@@ -35,8 +35,8 @@ RUNTIME_IMAGE=$1
   && echo "To improve build times use a runtime image in the target repo" && exit 1
 
 # crane hangs for a long time if it doesn't know that the registry is plain http
-RUNTIME_IMAGE=$(echo $RUNTIME_IMAGE | sed 's|.local|.local:80|')
-IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
+RUNTIME_IMAGE=$(echo $RUNTIME_IMAGE | sed 's|.local/|.local:80/|')
+IMAGE=$(echo $IMAGE | sed 's|.local/|.local:80/|')
 
 # This is a PoC, let's make a lot of assumptions to simplify
 context=.

From 34b78d0a40a54892e60c4ffc859b3f7aba8dfb52 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 10 Jun 2020 15:14:49 +0200
Subject: [PATCH 47/61] Includes agent in server, by commenting out

---
 docker-compose.dev-overrides.yml      | 12 ++++----
 docker-compose.test.yml               | 44 +++++++++++++--------------
 k3s/docker-ystack-proxy/entrypoint.sh |  2 +-
 3 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index 43952af3..b48043b7 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -4,12 +4,12 @@ services:
   server:
     ports:
     - 17143:17143
-
-  agent1:
-    ports:
-    # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead
-    #- 80:80
-    #- 443:443
+    #
+    # agent1:
+    #   ports:
+    #   # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead
+    #   #- 80:80
+    #   #- 443:443
     - "31710-31719:31710-31719"
     mem_limit: 8589934592
     memswap_limit: 0
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 5f74ec25..3f29fcff 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -19,7 +19,7 @@ services:
     command:
     - server
     - --https-listen-port=17143
-    - --disable-agent
+  #  - --disable-agent
     - --node-name=server
     - --no-deploy=traefik
     - --no-deploy=servicelb
@@ -43,31 +43,31 @@ services:
     mem_limit:  300000000
     memswap_limit: 0
 
-  agent1:
-    hostname: agent1
-    depends_on:
-    - server
-    build:
-      context: .
-      dockerfile: ./k3s/image/Dockerfile
-    tmpfs:
-    - /run
-    - /var/run
-    privileged: true
-    environment:
-    - K3S_URL=https://server:17143
-    - K3S_TOKEN=somethingtotallyrandom
-    expose:
-    - 8472
-    - 10250
-    # It's a feature that we keep ystack with kubernetes-assert light enough to fit these limits
-    mem_limit: 1500000000
-    memswap_limit: 0
+  # agent1:
+  #   hostname: agent1
+  #   depends_on:
+  #   - server
+  #   build:
+  #     context: .
+  #     dockerfile: ./k3s/image/Dockerfile
+  #   tmpfs:
+  #   - /run
+  #   - /var/run
+  #   privileged: true
+  #   environment:
+  #   - K3S_URL=https://server:17143
+  #   - K3S_TOKEN=somethingtotallyrandom
+  #   expose:
+  #   - 8472
+  #   - 10250
+  #   # It's a feature that we keep ystack with kubernetes-assert light enough to fit these limits
+  #   mem_limit: 1500000000
+  #   memswap_limit: 0
 
   ystack-proxy:
     depends_on:
     - server
-    - agent1
+  #  - agent1
     build:
       context: .
       dockerfile: ./k3s/docker-ystack-proxy/Dockerfile
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index 8b337337..b29aa3db 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -20,7 +20,7 @@ kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 
 [ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd
 
-NODE=agent1
+NODE=server
 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort})
 BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort})
 # Assuming ordering is predictable ...

From 07ca4613160a444cf063fd230a2d270eabb4c004 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 18 Jun 2020 13:23:23 +0200
Subject: [PATCH 48/61] This kubefwd release seems to work with Ubuntu 20.04

---
 bin/y-kubefwd | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/y-kubefwd b/bin/y-kubefwd
index 2ac79fa2..61f659e7 100755
--- a/bin/y-kubefwd
+++ b/bin/y-kubefwd
@@ -3,14 +3,14 @@
 set -e
 YBIN="$(dirname $0)"
 
-version=1.13.0
+version=1.14.0
 
 bin_name=kubefwd \
   bin_version=v${version} \
   Darwin_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_Darwin_amd64.tar.gz \
-  Darwin_sha256=49bce5f5914d81a478e301fae79ca63698b6eeaffcb38cf80bfc19150bf141e3 \
+  Darwin_sha256=1cb03198526192f21d632d531575ed91f6e7a58b5897bdb5814ed524c5e3d607 \
   Linux_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_linux_amd64.tar.gz \
-  Linux_sha256=4a6bc0f33bc0cde8738cc9611154702499d9581f3d1c0e65fb2d3951c1154ea5 \
+  Linux_sha256=f202a030aeaa6f9c5e0e6b764a0426a309662e81bab33505296c07391fc62b40 \
   bin_tgz_path=kubefwd \
   $YBIN/y-bin-dependency-download || exit $?
 

From b00c1153d719b008466c04dc352ae03474d5c75c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 17:49:04 +0200
Subject: [PATCH 49/61] Changes container name to match our earlier terminology

---
 README.md                                   | 13 ++++++-------
 bin/y-docker-compose                        | 14 ++++++++++++++
 docker-compose.dev-overrides.yml            |  2 +-
 docker-compose.test.yml                     | 14 +++++++-------
 k3s/{image => docker-image}/Dockerfile      |  4 ++--
 k3s/{image => docker-image}/registries.yaml |  0
 k3s/docker-ystack-proxy/entrypoint.sh       |  2 +-
 runner.Dockerfile.dockerignore              |  4 +---
 8 files changed, 32 insertions(+), 21 deletions(-)
 create mode 100755 bin/y-docker-compose
 rename k3s/{image => docker-image}/Dockerfile (76%)
 rename k3s/{image => docker-image}/registries.yaml (100%)

diff --git a/README.md b/README.md
index 2dcd5772..4c20ef80 100644
--- a/README.md
+++ b/README.md
@@ -97,14 +97,13 @@ docker volume rm ystack_admin 2> /dev/null || true
 
 ## Development
 
+Using the [y-docker-compose](./bin/y-docker-compose) wrapper that extends [docker-compose.test.yml](./docker-compose.test.yml) that is used for CI with [docker-compose.dev-overrides.yml](./docker-compose.dev-overrides.yml). The k3s [image](./k3s/docker-image/) is the stock k3s image with y-stack's local registry config.
+
 ```
-compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
-$compose down \
-  ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
-$compose up cleanup
-$compose up --build -d server
-$compose up --build -d agent1
-$compose up --build -d ystack-proxy
+y-docker-compose down
+y-docker-compose up --build -d master1
+#y-docker-compose up --build -d agent1
+y-docker-compose up --build -d ystack-proxy
 y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```
 
diff --git a/bin/y-docker-compose b/bin/y-docker-compose
new file mode 100755
index 00000000..1c8a7b02
--- /dev/null
+++ b/bin/y-docker-compose
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+[ -z "$YSTACK_HOME" ] && echo "Missing an YSTACK_HOME env" && exit 1
+
+COMPOSEFILES="-f $YSTACK_HOME/docker-compose.test.yml -f $YSTACK_HOME/docker-compose.dev-overrides.yml $COMPOSEFILES"
+docker-compose $COMPOSEFILES "$@"
+
+[ "$1" = "down" ] && \
+  echo "Cleaning up volumes" && \
+  (docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true) && \
+  echo "Removing local config" && \
+  (docker-compose $COMPOSEFILES up cleanup)
diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index b48043b7..4247454c 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -1,7 +1,7 @@
 version: '2.4'
 services:
 
-  server:
+  master1:
     ports:
     - 17143:17143
     #
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 3f29fcff..f6509905 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -12,15 +12,15 @@ services:
     volumes:
     - admin:/admin
 
-  server:
+  master1:
     build:
       context: .
-      dockerfile: ./k3s/image/Dockerfile
+      dockerfile: ./k3s/docker-image/Dockerfile
     command:
     - server
     - --https-listen-port=17143
   #  - --disable-agent
-    - --node-name=server
+    - --node-name=master1
     - --no-deploy=traefik
     - --no-deploy=servicelb
     - --kube-apiserver-arg
@@ -46,7 +46,7 @@ services:
   # agent1:
   #   hostname: agent1
   #   depends_on:
-  #   - server
+  #   - master1
   #   build:
   #     context: .
   #     dockerfile: ./k3s/image/Dockerfile
@@ -55,7 +55,7 @@ services:
   #   - /var/run
   #   privileged: true
   #   environment:
-  #   - K3S_URL=https://server:17143
+  #   - K3S_URL=https://master1:17143
   #   - K3S_TOKEN=somethingtotallyrandom
   #   expose:
   #   - 8472
@@ -66,7 +66,7 @@ services:
 
   ystack-proxy:
     depends_on:
-    - server
+    - master1
   #  - agent1
     build:
       context: .
@@ -114,7 +114,7 @@ services:
         echo "Waiting for a kubeconfig ..." && sleep 1
       done
 
-      cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
+      cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|master1|' > ~/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
       kubectl get nodes
 
diff --git a/k3s/image/Dockerfile b/k3s/docker-image/Dockerfile
similarity index 76%
rename from k3s/image/Dockerfile
rename to k3s/docker-image/Dockerfile
index 8b8305f5..e1eb35bb 100644
--- a/k3s/image/Dockerfile
+++ b/k3s/docker-image/Dockerfile
@@ -1,6 +1,6 @@
-FROM rancher/k3s:v1.18.3-k3s1@sha256:faddccf57fa09d7e11698eae56cbcf4f764ffa5a6917de9f4326af0b26c9b746
+FROM rancher/k3s:v1.18.4-k3s1@sha256:2555ed1896512b320c175859a95a5f1d6a2c64b501a84bc45d853a9bd0be0dfd
 
-COPY k3s/image/registries.yaml /etc/rancher/k3s/registries.yaml
+COPY k3s/docker-image/registries.yaml /etc/rancher/k3s/registries.yaml
 # The clusterIP should be predefined
 RUN sed -i 's|http://builds-registry.ystack.svc.cluster.local|http://10.43.0.50|' /etc/rancher/k3s/registries.yaml
 
diff --git a/k3s/image/registries.yaml b/k3s/docker-image/registries.yaml
similarity index 100%
rename from k3s/image/registries.yaml
rename to k3s/docker-image/registries.yaml
diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index b29aa3db..5597cc8c 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -8,7 +8,7 @@ until test -f /admin/.kube/kubeconfig.yaml; do
 done
 
 set -e
-cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
+cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|master1|' > ~/.kube/config
 kubectl-waitretry --for=condition=Ready node --all
 
 # Might speed up provision, due to the dependency minio -> registry -> builds, but should't be necessary
diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore
index 25c6218f..87513e2f 100644
--- a/runner.Dockerfile.dockerignore
+++ b/runner.Dockerfile.dockerignore
@@ -16,7 +16,7 @@ bin/*-bin
 !y-assert-support/
 
 # k3s configuration
-!k3s/image/registries.yaml
+!k3s/docker-*
 !k3s/0*
 !k3s/1*
 !k3s/2*
@@ -27,5 +27,3 @@ bin/*-bin
 !k3s/7*
 !k3s/8*
 !k3s/9*
-!k3s/docker-ystack-proxy/envoy.template.yaml
-!k3s/docker-ystack-proxy/entrypoint.sh

From 503b7402d3e3af01b18cfa30c0ab65fddfe7c6e7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 17:49:58 +0200
Subject: [PATCH 50/61] Fixes a wrapper script mistake

---
 bin/y-container-structure-test | 2 +-
 bin/y-kubie                    | 2 +-
 bin/y-kustomize                | 2 +-
 bin/y-minikube                 | 2 +-
 bin/y-skaffold                 | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/bin/y-container-structure-test b/bin/y-container-structure-test
index fd03c8a9..66d5fb17 100755
--- a/bin/y-container-structure-test
+++ b/bin/y-container-structure-test
@@ -12,4 +12,4 @@ bin_name=container-structure-test \
   Linux_sha256=09cb373a9e622a4120577391acb28c6b2aea55f883a4351db8e8de85f0cf8a5b \
   y-bin-dependency-download || exit $?
 
-y-container-structure-test-v${version}-bin "$@" || echo $?
+y-container-structure-test-v${version}-bin "$@" || exit $?
diff --git a/bin/y-kubie b/bin/y-kubie
index 20fdb764..58054ffc 100755
--- a/bin/y-kubie
+++ b/bin/y-kubie
@@ -16,4 +16,4 @@ command -v fzf > /dev/null || echo "Note that kubie recommends having fzf instal
 
 [ "$1" = "completion" ] && curl -sLS https://github.com/sbstp/kubie/raw/3ebbf728cc52a57dfd9913da120d47a44e783702/completion/kubie.bash && exit 0
 
-y-kubie-v${version}-bin "$@" || echo $?
+y-kubie-v${version}-bin "$@" || exit $?
diff --git a/bin/y-kustomize b/bin/y-kustomize
index 8d0a1b93..7b4bb2b0 100755
--- a/bin/y-kustomize
+++ b/bin/y-kustomize
@@ -12,4 +12,4 @@ bin_name=kustomize \
   Linux_sha256=a04d79a013827c9ebb0abfe9d41cbcedf507a0310386c8d9a7efec7a36f9d7a3 \
   y-bin-dependency-download || exit $?
 
-y-kustomize-v${version}-bin "$@" || echo $?
+y-kustomize-v${version}-bin "$@" || exit $?
diff --git a/bin/y-minikube b/bin/y-minikube
index 65f7aadb..0be7503a 100755
--- a/bin/y-minikube
+++ b/bin/y-minikube
@@ -11,4 +11,4 @@ bin_name=minikube \
   Darwin_sha256=5bda29e2d990bb8ac9da1767143e228772adc45507d22a49b5af70b03e7db682 \
   y-bin-dependency-download || exit $?
 
-y-minikube-v${version}-bin "$@" || echo $?
+y-minikube-v${version}-bin "$@" || exit $?
diff --git a/bin/y-skaffold b/bin/y-skaffold
index f55d62ae..6faa5171 100755
--- a/bin/y-skaffold
+++ b/bin/y-skaffold
@@ -28,4 +28,4 @@ grep 'current-context: minikube' $KUBECONFIG && {
 SKAFFOLD_NO_PRUNE=true \
 SKAFFOLD_UPDATE_CHECK=false \
   \
-  $YBIN/y-skaffold-v${version}-bin "$@" || echo $?
+  $YBIN/y-skaffold-v${version}-bin "$@" || exit $?

From 6e927464f3040b34eaba590af89e313eb72f9e03 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 17:53:23 +0200
Subject: [PATCH 51/61] Removes agent1 node config as it will quickly get
 outdated

---
 README.md                        |  1 -
 docker-compose.dev-overrides.yml |  6 ------
 docker-compose.test.yml          | 25 +------------------------
 3 files changed, 1 insertion(+), 31 deletions(-)

diff --git a/README.md b/README.md
index 4c20ef80..5ca3d521 100644
--- a/README.md
+++ b/README.md
@@ -102,7 +102,6 @@ Using the [y-docker-compose](./bin/y-docker-compose) wrapper that extends [docke
 ```
 y-docker-compose down
 y-docker-compose up --build -d master1
-#y-docker-compose up --build -d agent1
 y-docker-compose up --build -d ystack-proxy
 y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
 ```
diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index 4247454c..e35c3c2b 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -4,12 +4,6 @@ services:
   master1:
     ports:
     - 17143:17143
-    #
-    # agent1:
-    #   ports:
-    #   # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead
-    #   #- 80:80
-    #   #- 443:443
     - "31710-31719:31710-31719"
     mem_limit: 8589934592
     memswap_limit: 0
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index f6509905..332371bc 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -19,7 +19,6 @@ services:
     command:
     - server
     - --https-listen-port=17143
-  #  - --disable-agent
     - --node-name=master1
     - --no-deploy=traefik
     - --no-deploy=servicelb
@@ -40,34 +39,12 @@ services:
     volumes:
     - k3s-server:/var/lib/rancher/k3s
     - admin:/admin
-    mem_limit:  300000000
+    mem_limit:  1800000000
     memswap_limit: 0
 
-  # agent1:
-  #   hostname: agent1
-  #   depends_on:
-  #   - master1
-  #   build:
-  #     context: .
-  #     dockerfile: ./k3s/image/Dockerfile
-  #   tmpfs:
-  #   - /run
-  #   - /var/run
-  #   privileged: true
-  #   environment:
-  #   - K3S_URL=https://master1:17143
-  #   - K3S_TOKEN=somethingtotallyrandom
-  #   expose:
-  #   - 8472
-  #   - 10250
-  #   # It's a feature that we keep ystack with kubernetes-assert light enough to fit these limits
-  #   mem_limit: 1500000000
-  #   memswap_limit: 0
-
   ystack-proxy:
     depends_on:
     - master1
-  #  - agent1
     build:
       context: .
       dockerfile: ./k3s/docker-ystack-proxy/Dockerfile

From 555ce0e23307b17bba03d68561c1e5c0b73d35ee Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 17:59:56 +0200
Subject: [PATCH 52/61] Having a named context helps, even when it's alone in
 the yaml file

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5ca3d521..d7701fdb 100644
--- a/README.md
+++ b/README.md
@@ -103,7 +103,8 @@ Using the [y-docker-compose](./bin/y-docker-compose) wrapper that extends [docke
 y-docker-compose down
 y-docker-compose up --build -d master1
 y-docker-compose up --build -d ystack-proxy
-y-kubie ctx -f ./devcluster/.kube/kubeconfig.yaml
+kubectl --kubeconfig=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml config rename-context default ystack-local
+y-kubie ctx -f $YSTACK_HOME/devcluster/.kube/kubeconfig.yaml
 ```
 
 To add monitoring support run `y-cluster-assert-install`.

From 4d4e2042cf05379c5f9ce6a4bfb61b056773991a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 18:53:58 +0200
Subject: [PATCH 53/61] Another reference to master's hostname

---
 k3s/docker-ystack-proxy/entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh
index 5597cc8c..148f8e59 100755
--- a/k3s/docker-ystack-proxy/entrypoint.sh
+++ b/k3s/docker-ystack-proxy/entrypoint.sh
@@ -20,7 +20,7 @@ kubectl apply -f /var/lib/rancher/k3s/server/manifests/
 
 [ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd
 
-NODE=server
+NODE=master1
 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort})
 BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort})
 # Assuming ordering is predictable ...

From 2a24a237940a2235a45ae333bc7e94979bc8169b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 18:54:14 +0200
Subject: [PATCH 54/61] Fixes dev volume independent of CWD

---
 docker-compose.dev-overrides.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml
index e35c3c2b..e2f958ca 100644
--- a/docker-compose.dev-overrides.yml
+++ b/docker-compose.dev-overrides.yml
@@ -27,5 +27,5 @@ volumes:
     driver: local
     driver_opts:
       type: none
-      device: $PWD/devcluster
+      device: $YSTACK_HOME/devcluster
       o: bind

From 387faba1b238775e395d29a0f6b090b5d600b680 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 18:54:32 +0200
Subject: [PATCH 55/61] Scope creep y-docker-compose with some shortcuts

---
 bin/y-docker-compose | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/bin/y-docker-compose b/bin/y-docker-compose
index 1c8a7b02..fad86774 100755
--- a/bin/y-docker-compose
+++ b/bin/y-docker-compose
@@ -5,10 +5,22 @@ set -e
 [ -z "$YSTACK_HOME" ] && echo "Missing an YSTACK_HOME env" && exit 1
 
 COMPOSEFILES="-f $YSTACK_HOME/docker-compose.test.yml -f $YSTACK_HOME/docker-compose.dev-overrides.yml $COMPOSEFILES"
+
+KUBECONFIG=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml
+
+[ "$1" = "kubeconfig-path" ] && \
+  echo "$KUBECONFIG" && \
+  exit 0
+
+[ "$1" = "kubie" ] && \
+  y-kubie ctx -f $KUBECONFIG && \
+  exit 0
+
 docker-compose $COMPOSEFILES "$@"
 
 [ "$1" = "down" ] && \
+  echo "Removing local config" && \
+  (docker-compose $COMPOSEFILES up cleanup) && \
   echo "Cleaning up volumes" && \
   (docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true) && \
-  echo "Removing local config" && \
-  (docker-compose $COMPOSEFILES up cleanup)
+  echo "Done"

From bee8bdec64bb44dfd581265483e6612a31fbaadb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 20 Jun 2020 19:11:58 +0200
Subject: [PATCH 56/61] These 3 places refer to a kubernetes-assert commit

---
 bin/y-cluster-assert-install | 2 +-
 specs/kustomization.yaml     | 2 +-
 specs/skaffold.yaml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/y-cluster-assert-install b/bin/y-cluster-assert-install
index e6b45949..e9f730ee 100755
--- a/bin/y-cluster-assert-install
+++ b/bin/y-cluster-assert-install
@@ -3,7 +3,7 @@
 set -e
 
 OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
-KUBERNETES_ASSERT_VERSION=f0318952dd4990bb3d4c9d163f8044f13e2b647d
+KUBERNETES_ASSERT_VERSION=0054d175107ecd8363a4ddd68b6ad082f2a780b3
 
 ctx=$1
 case $ctx in
diff --git a/specs/kustomization.yaml b/specs/kustomization.yaml
index ed2dbdf3..93367146 100644
--- a/specs/kustomization.yaml
+++ b/specs/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=930ee2627ac53b7e356ce6a317226a010ffe3c8d
+- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=0054d175107ecd8363a4ddd68b6ad082f2a780b3
 namePrefix: ystack-
 commonLabels:
   app: ystack
diff --git a/specs/skaffold.yaml b/specs/skaffold.yaml
index 0835205c..1b799fd6 100644
--- a/specs/skaffold.yaml
+++ b/specs/skaffold.yaml
@@ -12,7 +12,7 @@ build:
     custom:
       buildCommand: >-
         y-build-like-sync
-        builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:825e4973566d3a736e3bbb689c533a33a3092173
+        builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:0054d175107ecd8363a4ddd68b6ad082f2a780b3
     sync:
       manual:
       - src: '**'

From c47460e35f462c0805c1a980684b7bf6ab328dc6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 21 Jun 2020 06:01:11 +0200
Subject: [PATCH 57/61] Allows nonroot users to modify appended files, such as
 assert specs

The chosen UID is for gcr.io/distroless nonroot, but the GID is nogroup
present in distro base images like ubuntu
---
 bin/y-build-like-sync | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync
index a19c46f9..74399efb 100755
--- a/bin/y-build-like-sync
+++ b/bin/y-build-like-sync
@@ -50,7 +50,7 @@ dest=$(echo $dest | sed 's|^/||')
 list=$(mktemp)
 (cd $context; git ls-files -c -o --exclude-standard -- .) > $list
 tar=$(mktemp)
-tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list
+tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list --mode='ug+rw' --group=65534 --owner=65532
 rm $list
 
 set -x

From 68378746d69eca88ba5e34e18fcd658fad65372f Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 21 Jun 2020 06:21:59 +0200
Subject: [PATCH 58/61] Upgrading kuberenetes-assert is now two git refs and an
 autobuild sha

---
 bin/y-cluster-assert-install | 2 +-
 specs/kustomization.yaml     | 2 +-
 specs/skaffold.yaml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/y-cluster-assert-install b/bin/y-cluster-assert-install
index e9f730ee..ccbb6085 100755
--- a/bin/y-cluster-assert-install
+++ b/bin/y-cluster-assert-install
@@ -3,7 +3,7 @@
 set -e
 
 OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
-KUBERNETES_ASSERT_VERSION=0054d175107ecd8363a4ddd68b6ad082f2a780b3
+KUBERNETES_ASSERT_VERSION=cae925e86004a4ecfe852aec3b54ee8f01c2063a
 
 ctx=$1
 case $ctx in
diff --git a/specs/kustomization.yaml b/specs/kustomization.yaml
index 93367146..ff4eff6f 100644
--- a/specs/kustomization.yaml
+++ b/specs/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=0054d175107ecd8363a4ddd68b6ad082f2a780b3
+- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=cae925e86004a4ecfe852aec3b54ee8f01c2063a
 namePrefix: ystack-
 commonLabels:
   app: ystack
diff --git a/specs/skaffold.yaml b/specs/skaffold.yaml
index 1b799fd6..b092cf91 100644
--- a/specs/skaffold.yaml
+++ b/specs/skaffold.yaml
@@ -12,7 +12,7 @@ build:
     custom:
       buildCommand: >-
         y-build-like-sync
-        builds-registry.ystack.svc.cluster.local/yolean/kubernetes-assert/runtime-nodejs:0054d175107ecd8363a4ddd68b6ad082f2a780b3
+        solsson/kubernetes-assert:runtime-nodejs@sha256:73ca64bdef6d9df38a8a5f5cef2d7d8375d9d14a77c304d0617c83ae69483bfd
     sync:
       manual:
       - src: '**'

From 139effc9eb4aed4ab7fa3eaa0df7a0289ba25db5 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 21 Jun 2020 07:27:20 +0200
Subject: [PATCH 59/61] Support CI builds with y-build-like-sync

---
 bin/y-build-like-sync   | 2 +-
 docker-compose.test.yml | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync
index 74399efb..ec8eee4d 100755
--- a/bin/y-build-like-sync
+++ b/bin/y-build-like-sync
@@ -48,7 +48,7 @@ dest=$(cat skaffold.yaml | grep 'dest:' | awk '{ print $2 }')
 dest=$(echo $dest | sed 's|^/||')
 
 list=$(mktemp)
-(cd $context; git ls-files -c -o --exclude-standard -- .) > $list
+(cd $context; git ls-files -c -o --exclude-standard -- . || find . -type f) > $list
 tar=$(mktemp)
 tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list --mode='ug+rw' --group=65534 --owner=65532
 rm $list
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 332371bc..64a49b9f 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -64,6 +64,7 @@ services:
     links:
     - ystack-proxy:builds-registry.ystack.svc.cluster.local
     - ystack-proxy:buildkitd.ystack.svc.cluster.local
+    - ystack-proxy:monitoring.ystack.svc.cluster.local
     build:
       context: .
       # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore

From 4ed0891bcf801f16b567ee83245b19d0420ded6e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 21 Jun 2020 08:24:23 +0200
Subject: [PATCH 60/61] Three refs to a kubernetes-assert commit, again

---
 bin/y-cluster-assert-install | 2 +-
 specs/kustomization.yaml     | 2 +-
 specs/skaffold.yaml          | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/bin/y-cluster-assert-install b/bin/y-cluster-assert-install
index ccbb6085..e7398d13 100755
--- a/bin/y-cluster-assert-install
+++ b/bin/y-cluster-assert-install
@@ -3,7 +3,7 @@
 set -e
 
 OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293
-KUBERNETES_ASSERT_VERSION=cae925e86004a4ecfe852aec3b54ee8f01c2063a
+KUBERNETES_ASSERT_VERSION=41207ccf8536ce022919d9aef86d2124b2c142b1
 
 ctx=$1
 case $ctx in
diff --git a/specs/kustomization.yaml b/specs/kustomization.yaml
index ff4eff6f..eacf2dae 100644
--- a/specs/kustomization.yaml
+++ b/specs/kustomization.yaml
@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
-- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=cae925e86004a4ecfe852aec3b54ee8f01c2063a
+- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=41207ccf8536ce022919d9aef86d2124b2c142b1
 namePrefix: ystack-
 commonLabels:
   app: ystack
diff --git a/specs/skaffold.yaml b/specs/skaffold.yaml
index b092cf91..bfb67b9e 100644
--- a/specs/skaffold.yaml
+++ b/specs/skaffold.yaml
@@ -12,7 +12,7 @@ build:
     custom:
       buildCommand: >-
         y-build-like-sync
-        solsson/kubernetes-assert:runtime-nodejs@sha256:73ca64bdef6d9df38a8a5f5cef2d7d8375d9d14a77c304d0617c83ae69483bfd
+        yolean/assert-runtime-nodejs:41207ccf8536ce022919d9aef86d2124b2c142b1@sha256:8c44b3817e81c4d6cb01b6decbffb7628b701a5b860161263b008f9d45646e35
     sync:
       manual:
       - src: '**'

From bf846d4803b392661d9b55d39df64f1139c0bc15 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 21 Jun 2020 08:41:51 +0200
Subject: [PATCH 61/61] Helps us detect y-cluster-assert-install issues

---
 specs/monitoring.spec.js | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/specs/monitoring.spec.js b/specs/monitoring.spec.js
index b05e752f..4792608b 100644
--- a/specs/monitoring.spec.js
+++ b/specs/monitoring.spec.js
@@ -12,6 +12,22 @@ describe("Monitoring", () => {
       expect(alerts).toBeInstanceOf(Array);
     });
 
+    it("Prometheus has the kubernetes-assert PodMonitor", async () => {
+      const config = await fetch('http://monitoring.ystack:9090/api/v1/status/config').then(res => res.json());
+      expect(config).toHaveProperty('data.yaml');
+      expect(config.data.yaml).toMatch(/job_name: monitoring\/kubernetes-assert\/0/);
+    });
+
+    it("Prometheus finds at lest one target (this one) for the PodMonitor", async () => {
+      const targets = await fetch('http://monitoring.ystack:9090/api/v1/targets?state=active').then(res => res.json());
+      expect(targets).toHaveProperty('data.activeTargets');
+      expect(targets.data.activeTargets).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({scrapePool: 'monitoring/kubernetes-assert/0'})
+        ])
+      );
+    });
+
   });
 
 });