diff --git a/.dockerignore b/.dockerignore new file mode 120000 index 00000000..3f5a7b5b --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +runner.Dockerfile.dockerignore \ No newline at end of file diff --git a/bin/kubectl-waitretry b/bin/kubectl-waitretry new file mode 100755 index 00000000..9426c790 --- /dev/null +++ b/bin/kubectl-waitretry @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +[ -z "$WAITRETRIES" ] && WAITRETRIES=10 + +SLEEP=1 +RETRY=0 +until kubectl wait $@; do + [ $RETRY -lt $WAITRETRIES ] || exit 1 + echo "waitretry after $SLEEP seconds ..." + sleep $SLEEP + SLEEP=$(( $SLEEP * 2 )) + RETRY=$(( $RETRY + 1 )) +done diff --git a/bin/y-cluster-install-prometheus-operator b/bin/y-cluster-install-prometheus-operator new file mode 100755 index 00000000..3e1dd0d7 --- /dev/null +++ b/bin/y-cluster-install-prometheus-operator @@ -0,0 +1,7 @@ +#!/bin/sh +[ -z "$DEBUG" ] || set -x +set -e + +OPERATOR_VERSION=ab1f8da8d9195339266742dcef535df5b51522e1 + +kubectl -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke index 11dcb7e2..4d206d26 100755 --- a/bin/y-cluster-provision-gke +++ b/bin/y-cluster-provision-gke @@ -137,3 +137,5 @@ EOF kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \ gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \ echo "Warning: failed to create builds registry storage bucket" + +y-cluster-install-prometheus-operator diff --git a/bin/y-cluster-provision-k3s-docker b/bin/y-cluster-provision-k3s-docker new file mode 100755 index 00000000..8289d4b3 --- /dev/null +++ b/bin/y-cluster-provision-k3s-docker @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +[ -z "$KUBECONFIG" ] && echo "Provision requires an explicit KUBECONFIG env" && exit 1 + +COMPOSE="docker-compose -p ystack -f docker-compose.test.yml" +[ -z "$NODES" ] && NODES=2 +# TODO condition to toggle dev/qa mode +false || { + COMPOSE="$COMPOSE -f docker-compose.dev-overrides.yml" + # ATM with port-forward we only support one node + [ $NODES -eq 1 ] || echo "# Dev mode only supports one node, util we add some kind of load balancer container" + NODES=1 +} + +(cd $YSTACK_HOME/ && \ + $COMPOSE up --scale node=$NODES -d --build && \ + docker ps && \ + echo "# Compose commmand:" && echo "$COMPOSE" && \ + echo "# To tear down:" && echo "$COMPOSE down --remove-orphans -v" && \ + echo "# Currently, because sut doesn't wait for server to be up, you need another:" && echo "$COMPOSE up -d" && \ + true ) + +$COMPOSE exec server sh -c 'until test -f /admin/.kube/kubeconfig.yaml; do echo "# Waiting ..." && sleep 1; done; cat /admin/.kube/kubeconfig.yaml' \ + > "$KUBECONFIG.tmp" + +KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local + +y-kubeconfig-import "$KUBECONFIG.tmp" diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass index 9b2063db..4c40862a 100755 --- a/bin/y-cluster-provision-k3s-multipass +++ b/bin/y-cluster-provision-k3s-multipass @@ -90,6 +90,8 @@ KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server +KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator + y-kubeconfig-import "$KUBECONFIG.tmp" echo "# Done. Master IP: $K3S_NODEIP_MASTER. The cluster should be ready for y-stack converge now." diff --git a/bin/y-kubefwd b/bin/y-kubefwd index c3f549b8..63783efa 100755 --- a/bin/y-kubefwd +++ b/bin/y-kubefwd @@ -3,12 +3,14 @@ set -e YBIN="$(dirname $0)" -version=1.9.2 +version=1.9.6 bin_name=kubefwd \ bin_version=v${version} \ - Darwin_url=https://github.com/txn2/kubefwd/releases/download/v${version}/kubefwd_macOS_amd64.tar.gz \ - Darwin_sha256=3f49349f7de2c8f60ae4d1248be4f936c9082d2b23cf9fa093caef20a60685d4 \ + Darwin_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_Darwin_amd64.tar.gz \ + Darwin_sha256=8a9a75d5377fbc8fc83152cdab4319b0e72f71ef084ef07703713b63e3ca3388 \ + Linux_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_linux_amd64.tar.gz \ + Linux_sha256=4a37c2f9763b46d2aba25cb6d07be36775213e208525911df27e398ea7d94d31 \ bin_tgz_path=kubefwd \ y-bin-dependency-download || exit $? diff --git a/bin/y-skaffold b/bin/y-skaffold index a30fb2c9..aa4b0dec 100755 --- a/bin/y-skaffold +++ b/bin/y-skaffold @@ -5,14 +5,14 @@ YBIN="$(dirname $0)" [ "$1" = "dev" ] && echo "y-skaffold dev requires -n [namespace] as initial args" && exit 1 -version=0.40.0 +version=1.0.0 bin_name=skaffold \ bin_version=v${version} \ Darwin_url=https://github.com/GoogleContainerTools/skaffold/releases/download/v${version}/skaffold-darwin-amd64 \ - Darwin_sha256=5175e11b36009cf5c1ba53f1564a2c1e66fd5a48a87e73a9bb672bade6c51174 \ + Darwin_sha256=5a7b4dad8dfa83129c2076e799fa5a1a004f8c40cae0bb024828d5b542a96a84 \ Linux_url=https://github.com/GoogleContainerTools/skaffold/releases/download/v${version}/skaffold-linux-amd64 \ - Linux_sha256=f2d876e60cfc0dfb0fc79263bc9900e9c08a8300bf1e4b2bbfef3f9936cc78e0 \ + Linux_sha256=f03cb2115cd510b6de8b024165b6621e453935faaf99b20c8cdf9933eec9b913 \ y-bin-dependency-download || exit $? # https://skaffold.dev/docs/concepts/#local-development diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml new file mode 100644 index 00000000..17666ca2 --- /dev/null +++ b/docker-compose.dev-overrides.yml @@ -0,0 +1,17 @@ +version: '2.4' +services: + + server: + ports: + - 6443:6443 + + node: + ports: + - 80:30080 + - 443:30443 + mem_limit: 8589934592 + memswap_limit: 0 + + sut: + environment: + - KEEP_RUNNING=true diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..abd38fb5 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,75 @@ +version: '2.4' +services: + + server: + build: ./k3s/image + command: + - server + - --disable-agent + - -v=0 + environment: + - K3S_CLUSTER_SECRET=somethingtotallyrandom + - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml + - K3S_KUBECONFIG_MODE=666 + volumes: + - k3s-server:/var/lib/rancher/k3s + - admin:/admin + # Host port not needed for CI + #ports: + #- 6443:6443 + mem_limit: 300000000 + memswap_limit: 0 + + node: + build: ./k3s/image + tmpfs: + - /run + - /var/run + privileged: true + environment: + - K3S_URL=https://server:6443 + - K3S_CLUSTER_SECRET=somethingtotallyrandom + # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET + #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c + mem_limit: 1500000000 + memswap_limit: 0 + + sut: + depends_on: + - server + - node + build: + context: . + # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore + dockerfile: runner.Dockerfile + environment: + - KUBECONFIG=/admin/.kube/config + - KUBECONFIG_WAIT=30 + - KEEP_RUNNING=false + volumes: + - admin:/admin + entrypoint: + - /bin/bash + - -cex + command: + - | + until test -f /admin/.kube/kubeconfig.yaml; do + [ $$KUBECONFIG_WAIT -gt 0 ] || exit ${BULID_EXIT_CODE_ON_NO_CLUSTER:-0} + KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 )) + echo "Waiting for a kubeconfig ..." && sleep 1 + done + cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > /admin/.kube/config + kubectl-waitretry --for=condition=Ready node --all + kubectl apply -k ./metrics-server/ + kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack + y-cluster-install-prometheus-operator + if [ "$$KEEP_RUNNING" = "true" ]; then + echo "Will stay running for manual work" + sleep infinity + fi + mem_limit: 80000000 + memswap_limit: 0 + +volumes: + k3s-server: {} + admin: {} diff --git a/docker/dockerd-statefulset.yaml b/docker/dockerd-statefulset.yaml index d75c1e18..a2185ef3 100644 --- a/docker/dockerd-statefulset.yaml +++ b/docker/dockerd-statefulset.yaml @@ -32,7 +32,7 @@ spec: ln -s /var/lib/docker/overlay2/var-lib-docker-image /var/lib/docker/image containers: - name: dockerd - image: docker:19.03.3-dind@sha256:5d11c2a1300b7438e21cbb5b2bd704e663b70534abe90c9072081dfb18e144de + image: docker:19.03.4-dind@sha256:156e4e723c80223298259dad9a925a71f385456b731c139bec12d29b3a74170c securityContext: privileged: true args: diff --git a/examples/basic-dev-inner-loop/skaffold.yaml b/examples/basic-dev-inner-loop/skaffold.yaml index 5d659be7..0e0a2c3b 100644 --- a/examples/basic-dev-inner-loop/skaffold.yaml +++ b/examples/basic-dev-inner-loop/skaffold.yaml @@ -1,4 +1,4 @@ -apiVersion: skaffold/v1beta16 +apiVersion: skaffold/v1 kind: Config build: tagPolicy: diff --git a/k3s/image/Dockerfile b/k3s/image/Dockerfile new file mode 100644 index 00000000..ed0da34a --- /dev/null +++ b/k3s/image/Dockerfile @@ -0,0 +1,3 @@ +FROM rancher/k3s:v0.10.2@sha256:e1f359b0adf3e11db49e9b2c286382f44d8caebde567931eca92e6a9a12153db + +COPY registries.yaml /etc/rancher/k3s/registries.yaml diff --git a/k3s/image/registries.yaml b/k3s/image/registries.yaml new file mode 100644 index 00000000..423ef968 --- /dev/null +++ b/k3s/image/registries.yaml @@ -0,0 +1,7 @@ +mirrors: + "builds-registry.ystack.svc.cluster.local": + endpoint: + - http://builds-registry.ystack.svc.cluster.local + "prod-registry.ystack.svc.cluster.local": + endpoint: + - http://prod-registry.ystack.svc.cluster.local diff --git a/runner.Dockerfile b/runner.Dockerfile index d486be39..8be62c67 100644 --- a/runner.Dockerfile +++ b/runner.Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:19.10@sha256:a21f154506cc00974f647e13dbba6b7035da35c7669a4bb919515d895221face +FROM ubuntu:19.10@sha256:a5193c15d7705bc2be91781355c4932321c06c18914facdd113d5bfcace7f92d RUN set -ex; \ export DEBIAN_FRONTEND=noninteractive; \ @@ -12,10 +12,10 @@ RUN set -ex; \ RUN set -ex; \ F=$(mktemp); \ - curl -SLs https://dl.k8s.io/v1.16.1/kubernetes-client-linux-amd64.tar.gz \ + curl -SLs https://dl.k8s.io/v1.16.2/kubernetes-client-linux-amd64.tar.gz \ | tee $F \ | tar xzf - --strip-components=3 -C /usr/local/bin/; \ - echo "e355a74a17d96785b0b217673e67fa0f02daa1939f10d410602ac0a0d061a4db71d727b67f75aa886007dab95dd5c7f8cc38253d291dc4d2504ce673df69fb32 $F" \ + echo "69bb92c9b16c0286d7401d87cc73b85c88d6f9a17d2cf1748060e44525194b5be860daf7554c4c6319a546c9ff10f2b4df42a27e32d8f95a4052993b17ef57c0 $F" \ | sha512sum -c -; \ rm $F @@ -37,3 +37,4 @@ COPY bin/y-skaffold /usr/local/src/ystack/bin/ RUN y-skaffold COPY . /usr/local/src/ystack +WORKDIR /usr/local/src/ystack diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore index 64a08b49..53910b47 100644 --- a/runner.Dockerfile.dockerignore +++ b/runner.Dockerfile.dockerignore @@ -1,3 +1,15 @@ * !bin/y-* bin/*-bin +!bin/kubectl-* + +# Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date: +!metrics-server/ +!converge-generic/ +!minio/ +!registry/generic/ +!registry/node-update-hosts/ +!registry/generic,minio/ +!buildkit/ +!monitoring/rbac-base/ +!monitoring/instance-now/ diff --git a/test.sh b/test.sh new file mode 100755 index 00000000..354992ea --- /dev/null +++ b/test.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# What we think Docker Hub is running +BULID_EXIT_CODE_ON_NO_CLUSTER=1 docker-compose -f docker-compose.test.yml up --exit-code-from sut --scale node=1 sut +RESULT=$? +docker-compose -f docker-compose.test.yml down --remove-orphans -v + +echo "Result: $RESULT" +exit $RESULT