diff --git a/README.md b/README.md index 1474d661..4242b59f 100644 --- a/README.md +++ b/README.md @@ -87,3 +87,21 @@ Still doesn't guard against mistakes, because `kubectl -n ystack-dev delete pod` Y-stack is opinionated on Kubernetes devops tooling as well. We therefore download some CLIs to the aforementioned `PATH` entry. + +## CI test suite + +``` +docker volume rm ystack_admin 2> /dev/null || true +./test.sh +``` + +## Development + +``` +compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml' +$compose down \ + ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true +sudo rm test/.kube/kubeconfig.yaml +$compose up --build -d ystack-proxy +export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml +``` diff --git a/bin/y-assert b/bin/y-assert new file mode 100755 index 00000000..c7f79526 --- /dev/null +++ b/bin/y-assert @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +function log { + date -u +"%Y-%m-%dT%H:%M:%SZ" +} + +[ "$1" = "help" ] && echo ' +A convention for running https://github.com/Yolean/kubernetes-assert +using ephemeral namespaces. +' && exit 0 + +[ -z "$DIR" ] && DIR="$PWD" && echo "$(log) Module dir: $DIR" + +[ ! -f $DIR/skaffold.yaml ] && echo "Missing a skaffold.yaml in project dir $DIR" && exit 1 + +[ -z "$PROJECT" ] && PROJECT=$(echo $DIR | awk -F/ '{print $NF}' | tr -cd [:alnum:]- | tr '[:upper:]' '[:lower:]') + +[ -z "$ID" ] && ID=assert-$(date -u +"%Y%m%dt%H%M%Sz")-${PROJECT:0:10} && echo "$(log) Generated test ID: $ID" + +# What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor +TTL="janitor/ttl: 23h" + +ctx=$1 +case $ctx in + "--context="*) shift 1 ;; + *) + [ "$CI" != "true" ] && echo "First arg should be --context" && exit 1 + ctx="--context=$(kubectl config current-context)" && echo "$(log) Current ctx: $ctx" + ;; +esac +skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|') + +CIRUN=$1 +[ -z "$CIRUN" ] && CIRUN="run" + +# --- Up to this point we didn't actually do anything in any cluster, from here on we will --- + +if [ "$CI" = true ]; then + [ -f /run/secrets/kubernetes.io/serviceaccount/namespace ] && namespace=$(cat /run/secrets/kubernetes.io/serviceaccount/namespace) + [ -z "$namespace" ] && namespace=$(kubectl $ctx config view --minify --output 'jsonpath={..namespace}') + [ -z "$namespace" ] && echo "$(log) Failed to find a current namespace for CI=true mode" && exit 1 +else + namespace=$ID + echo "$(log) Creating (in $ctx) temp namespace: $namespace" + cat < ~/.kube/config kubectl-waitretry --for=condition=Ready node --all kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack + y-cluster-install-prometheus-operator + + cd /usr/local/src/ystack/examples/basic-dev-inner-loop + y-skaffold run + if [ "$$KEEP_RUNNING" = "true" ]; then echo "Will stay running for manual work" sleep infinity diff --git a/examples/basic-dev-inner-loop/backend/Dockerfile b/examples/basic-dev-inner-loop/backend/Dockerfile index 7cc39a1f..c195c42d 100644 --- a/examples/basic-dev-inner-loop/backend/Dockerfile +++ b/examples/basic-dev-inner-loop/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.16.3-alpine@sha256:77c898d0da5e7bfb6e05c9a64de136ba4e03889a72f3c298e95df822a38f450d +FROM node:12.16.3-alpine@sha256:84f5c9090cc741421b2376b2393e7d5fa2f89d36167c12eb57aef16dd64ecf6c ARG build_tag ARG build_timestamp diff --git a/examples/basic-dev-inner-loop/skaffold.yaml b/examples/basic-dev-inner-loop/skaffold.yaml index 46a5ea42..40243f56 100644 --- a/examples/basic-dev-inner-loop/skaffold.yaml +++ b/examples/basic-dev-inner-loop/skaffold.yaml @@ -1,4 +1,4 @@ -apiVersion: skaffold/v2alpha2 +apiVersion: skaffold/v2beta4 kind: Config build: tagPolicy: diff --git a/examples/in-cluster-build/Dockerfile b/examples/in-cluster-build/Dockerfile new file mode 100644 index 00000000..72770a81 --- /dev/null +++ b/examples/in-cluster-build/Dockerfile @@ -0,0 +1,5 @@ +FROM busybox + +COPY index.html /var/www/ + +ENTRYPOINT [ "busybox", "httpd", "-f", "-p", "8080", "-h", "/var/www/" ] diff --git a/examples/in-cluster-build/index.html b/examples/in-cluster-build/index.html new file mode 100644 index 00000000..8fa09a01 --- /dev/null +++ b/examples/in-cluster-build/index.html @@ -0,0 +1,6 @@ + + +With skaffold dev --port-forward=true you can edit this file locally +and just reload the page. + + diff --git a/examples/in-cluster-build/k8s/example-server-deployment.yaml b/examples/in-cluster-build/k8s/example-server-deployment.yaml new file mode 100644 index 00000000..b990d999 --- /dev/null +++ b/examples/in-cluster-build/k8s/example-server-deployment.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-server +spec: + selector: + matchLabels: + app: example-server + template: + metadata: + labels: + app: example-server + spec: + containers: + - name: busybox + image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server + ports: + - containerPort: 8080 diff --git a/examples/in-cluster-build/k8s/example-server-service.yaml b/examples/in-cluster-build/k8s/example-server-service.yaml new file mode 100644 index 00000000..1d7b5a7b --- /dev/null +++ b/examples/in-cluster-build/k8s/example-server-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-server +spec: + ports: + - port: 8080 + selector: + app: example-server diff --git a/examples/in-cluster-build/skaffold.yaml b/examples/in-cluster-build/skaffold.yaml new file mode 100644 index 00000000..8a6ad5a5 --- /dev/null +++ b/examples/in-cluster-build/skaffold.yaml @@ -0,0 +1,17 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: in-cluster-build +build: + artifacts: + - image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server + sync: + infer: ["**/*"] + kaniko: + reproducible: true + cluster: {} +deploy: + kubectl: + manifests: + - k8s/example-server-deployment.yaml + - k8s/example-server-service.yaml diff --git a/examples/sync-only/index.html b/examples/sync-only/index.html new file mode 100644 index 00000000..2444ff69 --- /dev/null +++ b/examples/sync-only/index.html @@ -0,0 +1,14 @@ + +Example sync + +This file will only be hosted (by the runtime container) after skaffold dev and modification. + +The example could be a bit lighter if: + + + diff --git a/examples/sync-only/k8s/static-http-deployment.yaml b/examples/sync-only/k8s/static-http-deployment.yaml new file mode 100644 index 00000000..a67035f2 --- /dev/null +++ b/examples/sync-only/k8s/static-http-deployment.yaml @@ -0,0 +1,45 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-http +spec: + selector: + matchLabels: + app: ystack-examples-sync-only + template: + metadata: + labels: + app: ystack-examples-sync-only + spec: + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + containers: + - name: busybox + image: busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 + ports: + - containerPort: 8080 + command: + - busybox + - httpd + - -f + - -p + - "8080" + - -h + - /var/www/ + volumeMounts: + - name: www-static + mountPath: /var/www + - name: content + image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-only + command: + - tail + - -f + - /dev/null + volumeMounts: + - name: www-static + mountPath: /var/www + volumes: + - name: www-static + emptyDir: {} diff --git a/examples/sync-only/k8s/static-http-service.yaml b/examples/sync-only/k8s/static-http-service.yaml new file mode 100644 index 00000000..2d0785e4 --- /dev/null +++ b/examples/sync-only/k8s/static-http-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: static-http +spec: + ports: + - port: 8080 + selector: + app: ystack-examples-sync-only diff --git a/examples/sync-only/skaffold.yaml b/examples/sync-only/skaffold.yaml new file mode 100644 index 00000000..6cbb162c --- /dev/null +++ b/examples/sync-only/skaffold.yaml @@ -0,0 +1,22 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: sync-only +build: + tagPolicy: + gitCommit: + variant: CommitSha + artifacts: + - image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-only + custom: + buildCommand: y-build-nothing + sync: + manual: + - src: '**' + dest: /var/www + cluster: {} +deploy: + kubectl: + manifests: + - k8s/static-http-service.yaml + - k8s/static-http-deployment.yaml diff --git a/k3s/00-ystack-namespace/kustomization.yaml b/k3s/00-ystack-namespace/kustomization.yaml new file mode 100644 index 00000000..6c95cac6 --- /dev/null +++ b/k3s/00-ystack-namespace/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ystack-namespace.yaml diff --git a/k3s/00-ystack-namespace/ystack-namespace.yaml b/k3s/00-ystack-namespace/ystack-namespace.yaml new file mode 100644 index 00000000..08b42f64 --- /dev/null +++ b/k3s/00-ystack-namespace/ystack-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ystack diff --git a/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml b/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml new file mode 100644 index 00000000..4cbb91b5 --- /dev/null +++ b/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: builds-registry +spec: + type: NodePort + clusterIP: 10.43.0.50 diff --git a/k3s/10-builds-registry-service/kustomization.yaml b/k3s/10-builds-registry-service/kustomization.yaml new file mode 100644 index 00000000..f23bbc00 --- /dev/null +++ b/k3s/10-builds-registry-service/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../../registry/builds-service +patchesStrategicMerge: +- builds-registry-magic-numbers.yaml diff --git a/k3s/docker-ystack-proxy/Dockerfile b/k3s/docker-ystack-proxy/Dockerfile new file mode 100644 index 00000000..80090525 --- /dev/null +++ b/k3s/docker-ystack-proxy/Dockerfile @@ -0,0 +1,23 @@ +FROM lachlanevenson/k8s-kubectl:v1.18.3@sha256:495f41912554fde10e41ac54e8df6fdbd0e859e893a4cab61c9224b296f3454d as kubectl + +COPY . /etc/ystack +RUN set -ex; \ + mkdir -p /var/lib/rancher/k3s/server/manifests; \ + for base in \ + 00-ystack-namespace \ + 10-builds-registry-service \ + registry,minio,buildkit; do \ + kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \ + done + +FROM envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89 + +COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl + +COPY bin/kubectl-waitretry /usr/local/bin/kubectl-waitretry + +COPY --from=kubectl /var/lib/rancher/k3s/server/manifests /var/lib/rancher/k3s/server/manifests + +COPY k3s/docker-ystack-proxy/envoy.template.yaml /envoy.template.yaml +COPY k3s/docker-ystack-proxy/entrypoint.sh /usr/local/bin/entrypoint +ENTRYPOINT [ "entrypoint" ] diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh new file mode 100755 index 00000000..50ae28e7 --- /dev/null +++ b/k3s/docker-ystack-proxy/entrypoint.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +mkdir ~/.kube +until test -f /admin/.kube/kubeconfig.yaml; do + [ $KUBECONFIG_WAIT -gt 0 ] || exit 1 + KUBECONFIG_WAIT=$(( $KUBECONFIG_WAIT - 1 )) + echo "Waiting for a kubeconfig ..." && sleep 1 +done + +set -e +cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config +kubectl-waitretry --for=condition=Ready node --all + +kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/ + +NODE=agent +REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort}) +BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort}) + +cat envoy.template.yaml \ + | sed "s|{{ node }}|$NODE|g" \ + | sed "s|{{ registry_nodeport }}|$REGISTRY|g" \ + | sed "s|{{ buildkit_nodeport }}|$BUILDKIT|g" \ + > /envoy.yaml + +# TODO do we pass on signals? +envoy --config-path /envoy.yaml diff --git a/k3s/docker-ystack-proxy/envoy.template.yaml b/k3s/docker-ystack-proxy/envoy.template.yaml new file mode 100644 index 00000000..c34e7c8b --- /dev/null +++ b/k3s/docker-ystack-proxy/envoy.template.yaml @@ -0,0 +1,64 @@ +admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 127.0.0.1 + port_value: 9903 +static_resources: + listeners: + - name: registry + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "registry" + - name: buildkitd + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 8547 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "buildkitd" + clusters: + - name: registry + type: STRICT_DNS + connect_timeout: 1s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_registry + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ registry_nodeport }} + - name: buildkitd + type: STRICT_DNS + connect_timeout: 30s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_buildkitd + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ buildkit_nodeport }} diff --git a/k3s/image/Dockerfile b/k3s/image/Dockerfile index 9b237e52..8b8305f5 100644 --- a/k3s/image/Dockerfile +++ b/k3s/image/Dockerfile @@ -1,3 +1,14 @@ -FROM rancher/k3s:v1.17.2-k3s1@sha256:674c68efbb2190284ee25c73c03fd9755fa77ad983e068b43542013a6bfe61de +FROM rancher/k3s:v1.18.3-k3s1@sha256:faddccf57fa09d7e11698eae56cbcf4f764ffa5a6917de9f4326af0b26c9b746 -COPY registries.yaml /etc/rancher/k3s/registries.yaml +COPY k3s/image/registries.yaml /etc/rancher/k3s/registries.yaml +# The clusterIP should be predefined +RUN sed -i 's|http://builds-registry.ystack.svc.cluster.local|http://10.43.0.50|' /etc/rancher/k3s/registries.yaml + +# Unfortunately /var/lib/rancher/k3s is a VOLUME so this has no effect on server start +# COPY registry/generic /etc/ystack/registry/generic +# COPY k3s /etc/ystack/k3s +# RUN set -ex; \ +# mkdir -p /var/lib/rancher/k3s/server/manifests; \ +# for base in 00-ystack-namespace in-docker-builds-registry; do \ +# kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \ +# done diff --git a/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml b/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml new file mode 100644 index 00000000..e6d36adc --- /dev/null +++ b/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: buildkitd-nodeport +spec: + type: NodePort + ports: + - port: 8547 + protocol: TCP + selector: + app: buildkitd + status: accepts-new-builds diff --git a/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml b/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml new file mode 100644 index 00000000..eaa1c7c6 --- /dev/null +++ b/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: buildkitd +spec: + replicas: 0 diff --git a/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml b/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml new file mode 100644 index 00000000..21ec56b7 --- /dev/null +++ b/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry +spec: + replicas: 1 diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/registry,minio,buildkit/kustomization.yaml new file mode 100644 index 00000000..4e47c13c --- /dev/null +++ b/k3s/registry,minio,buildkit/kustomization.yaml @@ -0,0 +1,10 @@ +bases: +- ../../buildkit +- ../../minio/standalone,defaultsecret +- ../../registry/generic,minio +resources: +- buildkitd-nodeport-service.yaml +patchesStrategicMerge: +- builds-registry-replicas-1.yaml +# Making it work with ystack-proxy is a todo, and maybe we could do some kind of scale up on demand +- buildkitd-replicas-0.yaml diff --git a/registry/generic/service-builds.yaml b/registry/builds-service/builds-registry-service.yaml similarity index 100% rename from registry/generic/service-builds.yaml rename to registry/builds-service/builds-registry-service.yaml diff --git a/registry/builds-service/kustomization.yaml b/registry/builds-service/kustomization.yaml new file mode 100644 index 00000000..92102610 --- /dev/null +++ b/registry/builds-service/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- builds-registry-service.yaml diff --git a/registry/generic,minio/kustomization.yaml b/registry/generic,minio/kustomization.yaml index 7e99d142..64881646 100644 --- a/registry/generic,minio/kustomization.yaml +++ b/registry/generic,minio/kustomization.yaml @@ -1,6 +1,5 @@ bases: - ../generic -- ../node-update-hosts resources: - bucket-create-ystack-builds.yaml patchesStrategicMerge: diff --git a/registry/generic/kustomization.yaml b/registry/generic/kustomization.yaml index 5be5a7d8..da0a800b 100644 --- a/registry/generic/kustomization.yaml +++ b/registry/generic/kustomization.yaml @@ -1,6 +1,4 @@ resources: -- service-builds.yaml -- service-prod-placeholder.yaml - deployment.yaml configMapGenerator: - name: registry-config diff --git a/registry/prod-service/kustomization.yaml b/registry/prod-service/kustomization.yaml new file mode 100644 index 00000000..d7f1a8c3 --- /dev/null +++ b/registry/prod-service/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- prod-registry-service-placeholder.yaml diff --git a/registry/generic/service-prod-placeholder.yaml b/registry/prod-service/prod-registry-service-placeholder.yaml similarity index 100% rename from registry/generic/service-prod-placeholder.yaml rename to registry/prod-service/prod-registry-service-placeholder.yaml diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore index 53910b47..cc62aead 100644 --- a/runner.Dockerfile.dockerignore +++ b/runner.Dockerfile.dockerignore @@ -3,13 +3,22 @@ bin/*-bin !bin/kubectl-* -# Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date: -!metrics-server/ -!converge-generic/ +#!metrics-server/ +#!converge-generic/ !minio/ +!registry/builds-service/ +!registry/prod-service/ !registry/generic/ !registry/node-update-hosts/ !registry/generic,minio/ !buildkit/ -!monitoring/rbac-base/ -!monitoring/instance-now/ +#!monitoring/rbac-base/ +#!monitoring/instance-now/ + +# k3s configuration +!k3s/image/registries.yaml +!k3s/00-ystack-namespace +!k3s/10-builds-registry-service +!k3s/registry,minio,buildkit +!k3s/docker-ystack-proxy/envoy.template.yaml +!k3s/docker-ystack-proxy/entrypoint.sh diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..eb386402 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +.kube