y-assert, y-build-nothing and split registry kustomize base

README.md

@@ -87,3 +87,21 @@ Still doesn't guard against mistakes, because `kubectl -n ystack-dev delete pod`
 
 Y-stack is opinionated on Kubernetes devops tooling as well.
 We therefore download some CLIs to the aforementioned `PATH` entry.
+
+## CI test suite
+
+```
+docker volume rm ystack_admin 2> /dev/null || true
+./test.sh
+```
+
+## Development
+
+```
+compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml'
+$compose down \
+  ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true
+sudo rm test/.kube/kubeconfig.yaml
+$compose up --build -d ystack-proxy
+export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml
+```

bin/y-assert

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+function log {
+  date -u +"%Y-%m-%dT%H:%M:%SZ"
+}
+
+[ "$1" = "help" ] && echo '
+A convention for running https://github.com/Yolean/kubernetes-assert
+using ephemeral namespaces.
+' && exit 0
+
+[ -z "$DIR" ] && DIR="$PWD" && echo "$(log) Module dir: $DIR"
+
+[ ! -f $DIR/skaffold.yaml ] && echo "Missing a skaffold.yaml in project dir $DIR" && exit 1
+
+[ -z "$PROJECT" ] && PROJECT=$(echo $DIR | awk -F/ '{print $NF}' | tr -cd [:alnum:]- | tr '[:upper:]' '[:lower:]')
+
+[ -z "$ID" ] && ID=assert-$(date -u +"%Y%m%dt%H%M%Sz")-${PROJECT:0:10} && echo "$(log) Generated test ID: $ID"
+
+# What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor
+TTL="janitor/ttl: 23h"
+
+ctx=$1
+case $ctx in
+  "--context="*) shift 1 ;;
+  *)
+    [ "$CI" != "true" ] && echo "First arg should be --context" && exit 1
+    ctx="--context=$(kubectl config current-context)" && echo "$(log) Current ctx: $ctx"
+    ;;
+esac
+skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|')
+
+CIRUN=$1
+[ -z "$CIRUN" ] && CIRUN="run"
+
+# --- Up to this point we didn't actually do anything in any cluster, from here on we will ---
+
+if [ "$CI" = true ]; then
+  [ -f /run/secrets/kubernetes.io/serviceaccount/namespace ] && namespace=$(cat /run/secrets/kubernetes.io/serviceaccount/namespace)
+  [ -z "$namespace" ] && namespace=$(kubectl $ctx config view --minify --output 'jsonpath={..namespace}')
+  [ -z "$namespace" ] && echo "$(log) Failed to find a current namespace for CI=true mode" && exit 1
+else
+  namespace=$ID
+  echo "$(log) Creating (in $ctx) temp namespace: $namespace"
+  cat <<EOF | kubectl $ctx apply -f -
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: $namespace
+  labels:
+    $TTL
+EOF
+fi
+skaffoldflags="$skaffoldflags -n $namespace"
+
+set -x
+y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN
+
+echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: */=/') -o name | xargs kubectl $ctx delete"

bin/y-build

@@ -67,9 +67,9 @@ fi
 
 echo "Build command:"
 set -x
-KUBECONFIG=$KUBECONFIG \
-BUILDKIT_HOST=$BUILDKIT_HOST \
-  y-buildctl build \
+y-buildctl \
+  --addr $BUILDKIT_HOST \
+  build \
     --frontend dockerfile.v0 \
     --local context="$BUILD_CONTEXT" \
     --local dockerfile="$BUILD_CONTEXT_DOCKERFILE" \

bin/y-build-nothing

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -e
+
+# Settings
+DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local
+[ -z "$BUILDS_REGISTRY" ] && BUILDS_REGISTRY=$DEFAULT_REGISTRY
+[ -z "$PUSH_REGISTRY" ]   && PUSH_REGISTRY=$DEFAULT_REGISTRY
+
+if [ "$(curl -s --connect-timeout 3 http://$BUILDS_REGISTRY/v2/)" != "{}" ]
+then
+  echo "ERROR Skaffold need local access to the builds registry for digest lookup"
+  echo "Registry: $BUILDS_REGISTRY"
+  echo "Look for y-stack's ingress or port-forward utilities"
+  exit 1
+fi
+
+[ -z "$IMAGE" ] && echo "No IMAGE env (from for example Skaffold)" && exit 1
+
+IMAGE=$IMAGE
+case "$IMAGE" in
+  $BUILDS_REGISTRY/* ) ;;
+  $PUSH_REGISTRY/* ) echo "Unlike y-build this script won't push to non-build registries" && exit 1 ;;
+  * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;;
+esac
+
+[ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx"
+
+CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9
+NOTHING_IMAGE=busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209
+
+BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz")
+
+# TODO copy is significantly faster if the source image is already in the target repo
+
+# crane hangs for a long time if it doesn't know that the registry is plain http
+NOTHING_IMAGE=$(echo $NOTHING_IMAGE | sed 's|.local|.local:80|')
+IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|')
+
+set -x
+kubectl $ctx -n ystack run --restart=Never --attach \
+  --image=$CRANE_IMAGE y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") -- \
+  cp $NOTHING_IMAGE $IMAGE

bin/y-k3s-install

@@ -21,8 +21,8 @@ mirrors:
     - http://prod-registry.ystack.svc.cluster.local
 EOF
 
-INSTALLER_REVISION=58ebdb2a2ec5318ca40649eb7bd31679cb679f71
-export INSTALL_K3S_VERSION=v1.17.5+k3s1
+INSTALLER_REVISION=16c8554b5ca6636dbacaf5e00cee31dfea87e622
+export INSTALL_K3S_VERSION=v1.16.10+k3s1
 curl -sfL https://github.com/rancher/k3s/raw/$INSTALLER_REVISION/install.sh | sh -
 
 service k3s start

converge-generic/kustomization.yaml

@@ -1,5 +1,8 @@
 namespace: ystack
 bases:
+- ../registry/builds-service
+- ../registry/prod-service
+- ../registry/node-update-hosts
 - ../buildkit
 - ../minio/standalone,defaultsecret
 #- ../registry/generic,kafka  # Note that kafka isn't a base (as it has its own namespace) and must be applied separately

docker-compose.dev-overrides.yml

@@ -3,15 +3,32 @@ services:
 
   server:
     ports:
-    - 6443:6443
+    - 17143:17143
 
-  node:
+  agent:
     ports:
-    - 80:30080
-    - 443:30443
+    # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead
+    #- 80:80
+    #- 443:443
+    - "31710-31719:31710-31719"
     mem_limit: 8589934592
     memswap_limit: 0
 
+  ystack-proxy:
+    ports:
+    # To run skaffold build/run/dev locally
+    - 80:80
+    # To support y-build as skaffold build (TODO - proxying fails)
+    - 8547:8547
+
   sut:
     environment:
     - KEEP_RUNNING=true
+
+volumes:
+  admin:
+    driver: local
+    driver_opts:
+      type: none
+      device: $PWD/test
+      o: bind

docker-compose.test.yml

@@ -2,42 +2,74 @@ version: '2.4'
 services:
 
   server:
-    build: ./k3s/image
+    build:
+      context: .
+      dockerfile: ./k3s/image/Dockerfile
     command:
     - server
+    - --https-listen-port=17143
     - --disable-agent
-    - -v=0
+    - --node-name=server
+    - --no-deploy=traefik
+    - --no-deploy=servicelb
+    - --kube-apiserver-arg
+    -   service-node-port-range=31710-31719
+    tmpfs:
+    - /run
+    - /var/run
+    privileged: true
     environment:
-    - K3S_CLUSTER_SECRET=somethingtotallyrandom
+    - K3S_TOKEN=somethingtotallyrandom
     - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml
     - K3S_KUBECONFIG_MODE=666
+    expose:
+    - 17143
+    - 8472
+    - 10250
     volumes:
     - k3s-server:/var/lib/rancher/k3s
     - admin:/admin
-    # Host port not needed for CI
-    #ports:
-    #- 6443:6443
     mem_limit:  300000000
     memswap_limit: 0
 
-  node:
-    build: ./k3s/image
+  agent:
+    depends_on:
+    - server
+    build:
+      context: .
+      dockerfile: ./k3s/image/Dockerfile
     tmpfs:
     - /run
     - /var/run
     privileged: true
     environment:
-    - K3S_URL=https://server:6443
-    - K3S_CLUSTER_SECRET=somethingtotallyrandom
-    # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET
-    #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c
+    - K3S_URL=https://server:17143
+    - K3S_TOKEN=somethingtotallyrandom
+    expose:
+    - 8472
+    - 10250
     mem_limit: 1500000000
     memswap_limit: 0
 
-  sut:
+  ystack-proxy:
     depends_on:
     - server
-    - node
+    - agent
+    build:
+      context: .
+      dockerfile: ./k3s/docker-ystack-proxy/Dockerfile
+    environment:
+    - KUBECONFIG_WAIT=30
+    expose:
+    - 80
+    - 8547
+    volumes:
+    - admin:/admin
+
+  sut:
+    links:
+    - ystack-proxy:builds-registry.ystack.svc.cluster.local
+    - ystack-proxy:buildkitd.ystack.svc.cluster.local
     build:
       context: .
       # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore
@@ -47,6 +79,7 @@ services:
     - KEEP_RUNNING=false
     volumes:
     - admin:/admin
+    - ./examples:/usr/local/src/ystack/examples
     entrypoint:
     - /bin/bash
     - -cx
@@ -62,7 +95,12 @@ services:
       cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config
       kubectl-waitretry --for=condition=Ready node --all
       kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack
+
       y-cluster-install-prometheus-operator
+
+      cd /usr/local/src/ystack/examples/basic-dev-inner-loop
+      y-skaffold run
+
       if [ "$$KEEP_RUNNING" = "true" ]; then
         echo "Will stay running for manual work"
         sleep infinity

examples/basic-dev-inner-loop/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:10.16.3-alpine@sha256:77c898d0da5e7bfb6e05c9a64de136ba4e03889a72f3c298e95df822a38f450d
+FROM node:12.16.3-alpine@sha256:84f5c9090cc741421b2376b2393e7d5fa2f89d36167c12eb57aef16dd64ecf6c
 
 ARG build_tag
 ARG build_timestamp

examples/basic-dev-inner-loop/skaffold.yaml

@@ -1,4 +1,4 @@
-apiVersion: skaffold/v2alpha2
+apiVersion: skaffold/v2beta4
 kind: Config
 build:
   tagPolicy:

examples/in-cluster-build/Dockerfile

@@ -0,0 +1,5 @@
+FROM busybox
+
+COPY index.html /var/www/
+
+ENTRYPOINT [ "busybox", "httpd", "-f", "-p", "8080", "-h", "/var/www/" ]

examples/in-cluster-build/index.html

@@ -0,0 +1,6 @@
+<html>
+<body>
+With <code>skaffold dev --port-forward=true</code> you can edit this file locally
+and just reload <a href="http://localhost:8080">the page</a>.
+</body>
+</html>

examples/in-cluster-build/k8s/example-server-deployment.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example-server
+spec:
+  selector:
+    matchLabels:
+      app: example-server
+  template:
+    metadata:
+      labels:
+        app: example-server
+    spec:
+      containers:
+      - name: busybox
+        image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server
+        ports:
+        - containerPort: 8080

examples/in-cluster-build/k8s/example-server-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: example-server
+spec:
+  ports:
+  - port: 8080
+  selector:
+    app: example-server

examples/in-cluster-build/skaffold.yaml

@@ -0,0 +1,17 @@
+apiVersion: skaffold/v2beta4
+kind: Config
+metadata:
+  name: in-cluster-build
+build:
+  artifacts:
+  - image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server
+    sync:
+      infer: ["**/*"]
+    kaniko:
+      reproducible: true
+  cluster: {}
+deploy:
+  kubectl:
+    manifests:
+    - k8s/example-server-deployment.yaml
+    - k8s/example-server-service.yaml

examples/sync-only/index.html

@@ -0,0 +1,14 @@
+<html>
+<head><title>Example sync</title></head>
+<body>
+This file will only be hosted (by the runtime container) after <code>skaffold dev</code> and modification.
+
+The example could be a bit lighter if:
+<ul>
+<p>The <code>NOTHING_IMAGE</code>; was already in the target repo. Crane cp would be faster.</p>
+<p>The image had a nonroot <code>USER</code>. We could remove <code>securityContext</code>.</p>
+<p>The image had a sleep entrypoint. We could remove the <code>command</code>.</p>
+<p>With https://github.com/kubernetes/kubernetes/issues/58512 we wouldn't need tar in the image.</p>
+</ul>
+</body>
+</html>