diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..fa3df9b6 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root = true + +[*] +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 diff --git a/README.md b/README.md index 4242b59f..d7701fdb 100644 --- a/README.md +++ b/README.md @@ -97,11 +97,32 @@ docker volume rm ystack_admin 2> /dev/null || true ## Development +Using the [y-docker-compose](./bin/y-docker-compose) wrapper that extends [docker-compose.test.yml](./docker-compose.test.yml) that is used for CI with [docker-compose.dev-overrides.yml](./docker-compose.dev-overrides.yml). The k3s [image](./k3s/docker-image/) is the stock k3s image with y-stack's local registry config. + +``` +y-docker-compose down +y-docker-compose up --build -d master1 +y-docker-compose up --build -d ystack-proxy +kubectl --kubeconfig=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml config rename-context default ystack-local +y-kubie ctx -f $YSTACK_HOME/devcluster/.kube/kubeconfig.yaml +``` + +To add monitoring support run `y-cluster-assert-install`. + +For [dev loops](./examples/) and `y-assert` the docker stack replaces `y-kubefwd` (hard to use in CI) +with container ports. +You need `cat /etc/hosts | grep 127.0.0 | grep cluster.local` to have something like: ``` -compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml' -$compose down \ - ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true -sudo rm test/.kube/kubeconfig.yaml -$compose up --build -d ystack-proxy -export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml +127.0.0.1 builds-registry.ystack.svc.cluster.local +127.0.0.1 buildkitd.ystack.svc.cluster.local +127.0.0.1 monitoring.ystack.svc.cluster.local ``` + +Test using: +``` +curl http://builds-registry.ystack.svc.cluster.local/v2/ +curl http://monitoring.ystack.svc.cluster.local:9090/api/v1/alertmanagers | jq '.data.activeAlertmanagers[0]' +curl http://monitoring.ystack.svc.cluster.local:9093/api/v2/status +``` + +Start a dev loop for actual asserts using `cd specs; y-skaffold --cache-artifacts=false dev` and start editing specs/*.spec.js. Run `y-assert` for CI-like runs until completion. diff --git a/bin/.gitignore b/bin/.gitignore index cfb3f545..a3ee39fc 100644 --- a/bin/.gitignore +++ b/bin/.gitignore @@ -2,9 +2,14 @@ # our executables are symlinked to real names buildctl +container-structure-test +crane helm +kubie +kustomize minikube mkcert +promtool rio skaffold kubefwd diff --git a/bin/y-assert b/bin/y-assert index c7f79526..486cbdf9 100755 --- a/bin/y-assert +++ b/bin/y-assert @@ -22,6 +22,11 @@ using ephemeral namespaces. # What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor TTL="janitor/ttl: 23h" +[ -z "$MONITORING_HOST" ] && MONITORING_HOST=http://monitoring.ystack.svc.cluster.local + +! curl -f -s --connect-timeout 3 $MONITORING_HOST:9090/ >/dev/null && echo "Failed to access the Prometheus endpoint" && exit 1 +! curl -f -s --connect-timeout 3 $MONITORING_HOST:9093/ >/dev/null && echo "Failed to access the Alertmanager endpoint" && exit 1 + ctx=$1 case $ctx in "--context="*) shift 1 ;; @@ -32,6 +37,9 @@ case $ctx in esac skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|') +[ -z "$PROFILES" ] && PROFILES="y-assert" +skaffoldflags="$skaffoldflags -p $PROFILES" + CIRUN=$1 [ -z "$CIRUN" ] && CIRUN="run" @@ -55,7 +63,21 @@ EOF fi skaffoldflags="$skaffoldflags -n $namespace" -set -x +# At this point we should know how to identify relevant assert metrics +[ -z "$assertlabels" ] && assertlabels="namespace=\"$namespace\""; + y-skaffold $skaffoldflags --cache-artifacts=false $CIRUN +echo "$(log) Prometheus queries for test results will use labels: $assertlabels" echo "$(log) cleanup-ish: kubectl $ctx get namespace -l $(echo $TTL | sed 's/: */=/') -o name | xargs kubectl $ctx delete" + +query=assertions_failed{$assertlabels} + +[ -z "$INTERVAL" ] && INTERVAL=5 +# This loop will continuously list errors and must fail on http errors or absence of the metric +until curl -s --data "query=$query" $MONITORING_HOST:9090/api/v1/query \ + | jq -r '.data.result | if length > 0 then . else error("No results") end | .[] | . as {$metric, $values} | .value[1] | . as $vs | tonumber as $v | if $v > 0 then error($metric.namespace+" "+$metric.pod+" "+$vs) else empty end' +do sleep $INTERVAL +done + +echo "Zero failures now with: curl -s --data query=$query $MONITORING_HOST:9090/api/v1/query" diff --git a/bin/y-build-like-sync b/bin/y-build-like-sync new file mode 100755 index 00000000..ec8eee4d --- /dev/null +++ b/bin/y-build-like-sync @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +# Bash isn't a good tool for producing a tar from a build context +# but this script is an experiment on how to compose an image +# from a runtime and some static files using go-containerregistry's crane +# The user shouldn't need to write Dockerfile or .dockerignore, +# but we could probably use a generated Dockerfile + kaniko instead, or Buildpacks + +# Settings +DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local +[ -z "$BUILDS_REGISTRY" ] && BUILDS_REGISTRY=$DEFAULT_REGISTRY +[ -z "$PUSH_REGISTRY" ] && PUSH_REGISTRY=$DEFAULT_REGISTRY + +if [ "$(curl -s --connect-timeout 3 http://$BUILDS_REGISTRY/v2/)" != "{}" ] +then + echo "ERROR Skaffold need local access to the builds registry for digest lookup" + echo "Registry: $BUILDS_REGISTRY" + echo "Look for y-stack's ingress or port-forward utilities" + exit 1 +fi + +[ -z "$IMAGE" ] && echo "No IMAGE env (from for example Skaffold)" && exit 1 + +IMAGE=$IMAGE +case "$IMAGE" in + $BUILDS_REGISTRY/* ) ;; + $PUSH_REGISTRY/* ) echo "Unlike y-build this script won't push to non-build registries" && exit 1 ;; + * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;; +esac + +RUNTIME_IMAGE=$1 +[ -z "$RUNTIME_IMAGE" ] && echo "First argument must be a runtime image to append the layer to" \ + && echo "To improve build times use a runtime image in the target repo" && exit 1 + +# crane hangs for a long time if it doesn't know that the registry is plain http +RUNTIME_IMAGE=$(echo $RUNTIME_IMAGE | sed 's|.local/|.local:80/|') +IMAGE=$(echo $IMAGE | sed 's|.local/|.local:80/|') + +# This is a PoC, let's make a lot of assumptions to simplify +context=. +src='**' +# assuming a single manual sync, which is the reasonable use case for a runtime +[ ! -f skaffold.yaml ] && echo "This composition example assumes a sync defined in a skaffold.yaml" && exit 1 +dest=$(cat skaffold.yaml | grep 'dest:' | awk '{ print $2 }') +# this avoids "tar: Removing leading `/' from member names" and could come in handy if we can't use --transform +dest=$(echo $dest | sed 's|^/||') + +list=$(mktemp) +(cd $context; git ls-files -c -o --exclude-standard -- . || find . -type f) > $list +tar=$(mktemp) +tar --transform "s|^|$dest/|" --show-transformed-names -cvhf $tar -T $list --mode='ug+rw' --group=65534 --owner=65532 +rm $list + +set -x +y-crane append --insecure -b $RUNTIME_IMAGE -f $tar -t $IMAGE +set +x +rm $tar diff --git a/bin/y-build-nothing b/bin/y-build-nothing index 2b87082e..94c48d85 100755 --- a/bin/y-build-nothing +++ b/bin/y-build-nothing @@ -24,13 +24,8 @@ case "$IMAGE" in * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;; esac -[ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx" - -CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9 NOTHING_IMAGE=busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 -BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") - # TODO copy is significantly faster if the source image is already in the target repo # crane hangs for a long time if it doesn't know that the registry is plain http @@ -38,6 +33,4 @@ NOTHING_IMAGE=$(echo $NOTHING_IMAGE | sed 's|.local|.local:80|') IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|') set -x -kubectl $ctx -n ystack run --restart=Never --attach \ - --image=$CRANE_IMAGE y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") -- \ - cp $NOTHING_IMAGE $IMAGE +y-crane cp $NOTHING_IMAGE $IMAGE diff --git a/bin/y-cluster-assert-install b/bin/y-cluster-assert-install new file mode 100755 index 00000000..e7398d13 --- /dev/null +++ b/bin/y-cluster-assert-install @@ -0,0 +1,23 @@ +#!/bin/sh +[ -z "$DEBUG" ] || set -x +set -e + +OPERATOR_VERSION=1b525b8a77f79e08b8653101f622e73083daf293 +KUBERNETES_ASSERT_VERSION=41207ccf8536ce022919d9aef86d2124b2c142b1 + +ctx=$1 +case $ctx in + "--context="*) shift 1 ;; + *) echo "Initial arg must be --context=" && exit 1 ;; +esac + +# If we're to make this script idempotent we must make sure that re-applying the operator's bundle isn't painfully slow +kubectl $ctx create namespace monitoring + +kubectl $ctx -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml + +kubectl-waitretry $ctx -n default --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-operator + +kubectl $ctx -n monitoring apply -k github.com/Yolean/kubernetes-assert/example-small?ref=$KUBERNETES_ASSERT_VERSION + +kubectl-waitretry $ctx -n monitoring --for=condition=Ready pod --all diff --git a/bin/y-cluster-install-prometheus-operator b/bin/y-cluster-install-prometheus-operator deleted file mode 100755 index 3505dc68..00000000 --- a/bin/y-cluster-install-prometheus-operator +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -[ -z "$DEBUG" ] || set -x -set -e - -OPERATOR_VERSION=d0a871b710de7b764c05ced98dbd1eb32a681790 - -kubectl -n default apply -f https://github.com/coreos/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke index 30fae328..a262487f 100755 --- a/bin/y-cluster-provision-gke +++ b/bin/y-cluster-provision-gke @@ -94,19 +94,20 @@ esac gcloud container clusters create "$NAME" $FLAGS "$@" -GKE_CONTEXT_NAME=$(kubectl config current-context) +GKE_CONTEXT_NAME=$(kubectl $ctx config current-context) +ctx="--context=${GKE_CONTEXT_NAME}" echo "For outbound Internet access: y-cluster-provision-gke-network-outbound-enable" -kubectl create namespace ystack +kubectl $ctx create namespace ystack -kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/ +kubectl $ctx -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/ ### To be moved to registry provision script, but we do want the initial context name [ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds -kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME} +kubectl $ctx create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME} gcloud compute backend-buckets list gsutil ls @@ -120,8 +121,8 @@ metadata: spec: {} EOF -kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \ +kubectl $ctx apply -f gke/registry/builds-gcs-storagebucket.yaml || \ gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} || \ echo "Warning: failed to create builds registry storage bucket" -y-cluster-install-prometheus-operator +y-cluster-assert-install $ctx diff --git a/bin/y-cluster-provision-k3s-multipass b/bin/y-cluster-provision-k3s-multipass index 7c9d3af6..c64aa37a 100755 --- a/bin/y-cluster-provision-k3s-multipass +++ b/bin/y-cluster-provision-k3s-multipass @@ -26,11 +26,11 @@ multipass exec "$VM_NAME" -- sudo cat /etc/rancher/k3s/k3s.yaml \ KUBECONFIG="$KUBECONFIG.tmp" kubectl config rename-context default local -KUBECONFIG="$KUBECONFIG.tmp" kubectl create namespace ystack +KUBECONFIG="$KUBECONFIG.tmp" kubectl --context=local create namespace ystack #KUBECONFIG="$KUBECONFIG.tmp" kubectl apply -k $YSTACK_HOME/metrics-server -KUBECONFIG="$KUBECONFIG.tmp" y-cluster-install-prometheus-operator +KUBECONFIG="$KUBECONFIG.tmp" y-cluster-assert-install --context=local y-kubeconfig-import "$KUBECONFIG.tmp" diff --git a/bin/y-container-structure-test b/bin/y-container-structure-test index fd03c8a9..66d5fb17 100755 --- a/bin/y-container-structure-test +++ b/bin/y-container-structure-test @@ -12,4 +12,4 @@ bin_name=container-structure-test \ Linux_sha256=09cb373a9e622a4120577391acb28c6b2aea55f883a4351db8e8de85f0cf8a5b \ y-bin-dependency-download || exit $? -y-container-structure-test-v${version}-bin "$@" || echo $? +y-container-structure-test-v${version}-bin "$@" || exit $? diff --git a/bin/y-crane b/bin/y-crane new file mode 100755 index 00000000..6f538017 --- /dev/null +++ b/bin/y-crane @@ -0,0 +1,17 @@ +#!/bin/sh +[ -z "$DEBUG" ] || set -x +set -e +YBIN="$(dirname $0)" + +version=0.1.0 + +bin_name=crane \ + bin_version=v${version} \ + Darwin_url=https://github.com/google/go-containerregistry/releases/download/v${version}/go-containerregistry_${version}_Darwin_x86_64.tar.gz \ + Darwin_sha256=4bef595a9d0b6781b452b302317f6b70c72e43cce664f817fa777ff72f9ffe5f \ + Linux_url=https://github.com/google/go-containerregistry/releases/download/v${version}/go-containerregistry_${version}_Linux_x86_64.tar.gz \ + Linux_sha256=ee5bb44cc9097835d12e6b34e1c1e3399efb8ed3dfb68ee0f28c6e66952dc335 \ + bin_tgz_path=crane \ + $YBIN/y-bin-dependency-download || exit $? + +$YBIN/y-crane-v${version}-bin "$@" || exit $? diff --git a/bin/y-docker-compose b/bin/y-docker-compose new file mode 100755 index 00000000..fad86774 --- /dev/null +++ b/bin/y-docker-compose @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +[ -z "$YSTACK_HOME" ] && echo "Missing an YSTACK_HOME env" && exit 1 + +COMPOSEFILES="-f $YSTACK_HOME/docker-compose.test.yml -f $YSTACK_HOME/docker-compose.dev-overrides.yml $COMPOSEFILES" + +KUBECONFIG=$YSTACK_HOME/devcluster/.kube/kubeconfig.yaml + +[ "$1" = "kubeconfig-path" ] && \ + echo "$KUBECONFIG" && \ + exit 0 + +[ "$1" = "kubie" ] && \ + y-kubie ctx -f $KUBECONFIG && \ + exit 0 + +docker-compose $COMPOSEFILES "$@" + +[ "$1" = "down" ] && \ + echo "Removing local config" && \ + (docker-compose $COMPOSEFILES up cleanup) && \ + echo "Cleaning up volumes" && \ + (docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true) && \ + echo "Done" diff --git a/bin/y-kubefwd b/bin/y-kubefwd index 2ac79fa2..61f659e7 100755 --- a/bin/y-kubefwd +++ b/bin/y-kubefwd @@ -3,14 +3,14 @@ set -e YBIN="$(dirname $0)" -version=1.13.0 +version=1.14.0 bin_name=kubefwd \ bin_version=v${version} \ Darwin_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_Darwin_amd64.tar.gz \ - Darwin_sha256=49bce5f5914d81a478e301fae79ca63698b6eeaffcb38cf80bfc19150bf141e3 \ + Darwin_sha256=1cb03198526192f21d632d531575ed91f6e7a58b5897bdb5814ed524c5e3d607 \ Linux_url=https://github.com/txn2/kubefwd/releases/download/${version}/kubefwd_linux_amd64.tar.gz \ - Linux_sha256=4a6bc0f33bc0cde8738cc9611154702499d9581f3d1c0e65fb2d3951c1154ea5 \ + Linux_sha256=f202a030aeaa6f9c5e0e6b764a0426a309662e81bab33505296c07391fc62b40 \ bin_tgz_path=kubefwd \ $YBIN/y-bin-dependency-download || exit $? diff --git a/bin/y-kubie b/bin/y-kubie index 20fdb764..58054ffc 100755 --- a/bin/y-kubie +++ b/bin/y-kubie @@ -16,4 +16,4 @@ command -v fzf > /dev/null || echo "Note that kubie recommends having fzf instal [ "$1" = "completion" ] && curl -sLS https://github.com/sbstp/kubie/raw/3ebbf728cc52a57dfd9913da120d47a44e783702/completion/kubie.bash && exit 0 -y-kubie-v${version}-bin "$@" || echo $? +y-kubie-v${version}-bin "$@" || exit $? diff --git a/bin/y-kustomize b/bin/y-kustomize index 8d0a1b93..7b4bb2b0 100755 --- a/bin/y-kustomize +++ b/bin/y-kustomize @@ -12,4 +12,4 @@ bin_name=kustomize \ Linux_sha256=a04d79a013827c9ebb0abfe9d41cbcedf507a0310386c8d9a7efec7a36f9d7a3 \ y-bin-dependency-download || exit $? -y-kustomize-v${version}-bin "$@" || echo $? +y-kustomize-v${version}-bin "$@" || exit $? diff --git a/bin/y-minikube b/bin/y-minikube index 65f7aadb..0be7503a 100755 --- a/bin/y-minikube +++ b/bin/y-minikube @@ -11,4 +11,4 @@ bin_name=minikube \ Darwin_sha256=5bda29e2d990bb8ac9da1767143e228772adc45507d22a49b5af70b03e7db682 \ y-bin-dependency-download || exit $? -y-minikube-v${version}-bin "$@" || echo $? +y-minikube-v${version}-bin "$@" || exit $? diff --git a/bin/y-skaffold b/bin/y-skaffold index f55d62ae..6faa5171 100755 --- a/bin/y-skaffold +++ b/bin/y-skaffold @@ -28,4 +28,4 @@ grep 'current-context: minikube' $KUBECONFIG && { SKAFFOLD_NO_PRUNE=true \ SKAFFOLD_UPDATE_CHECK=false \ \ - $YBIN/y-skaffold-v${version}-bin "$@" || echo $? + $YBIN/y-skaffold-v${version}-bin "$@" || exit $? diff --git a/test/.gitignore b/devcluster/.gitignore similarity index 100% rename from test/.gitignore rename to devcluster/.gitignore diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml index 64d5a54a..e2f958ca 100644 --- a/docker-compose.dev-overrides.yml +++ b/docker-compose.dev-overrides.yml @@ -1,15 +1,9 @@ version: '2.4' services: - server: + master1: ports: - 17143:17143 - - agent: - ports: - # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead - #- 80:80 - #- 443:443 - "31710-31719:31710-31719" mem_limit: 8589934592 memswap_limit: 0 @@ -18,8 +12,11 @@ services: ports: # To run skaffold build/run/dev locally - 80:80 - # To support y-build as skaffold build (TODO - proxying fails) + # To support y-build as skaffold build - 8547:8547 + # To support y-assert + - 9090:9090 + - 9093:9093 sut: environment: @@ -30,5 +27,5 @@ volumes: driver: local driver_opts: type: none - device: $PWD/test + device: $YSTACK_HOME/devcluster o: bind diff --git a/docker-compose.test.yml b/docker-compose.test.yml index dfa958f6..64a49b9f 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -1,15 +1,25 @@ version: '2.4' services: - server: + cleanup: + image: busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 + command: + - sh + - -ce + - | + [ ! -f /admin/.kube/kubeconfig.yaml ] || rm -v /admin/.kube/kubeconfig.yaml + echo "Done. Exiting." + volumes: + - admin:/admin + + master1: build: context: . - dockerfile: ./k3s/image/Dockerfile + dockerfile: ./k3s/docker-image/Dockerfile command: - server - --https-listen-port=17143 - - --disable-agent - - --node-name=server + - --node-name=master1 - --no-deploy=traefik - --no-deploy=servicelb - --kube-apiserver-arg @@ -29,40 +39,24 @@ services: volumes: - k3s-server:/var/lib/rancher/k3s - admin:/admin - mem_limit: 300000000 - memswap_limit: 0 - - agent: - depends_on: - - server - build: - context: . - dockerfile: ./k3s/image/Dockerfile - tmpfs: - - /run - - /var/run - privileged: true - environment: - - K3S_URL=https://server:17143 - - K3S_TOKEN=somethingtotallyrandom - expose: - - 8472 - - 10250 - mem_limit: 1500000000 + mem_limit: 1800000000 memswap_limit: 0 ystack-proxy: depends_on: - - server - - agent + - master1 build: context: . dockerfile: ./k3s/docker-ystack-proxy/Dockerfile environment: - KUBECONFIG_WAIT=30 + # Scaled to zero by default because we'll want to introduce automated scale to demand + - BUILDKITD_REPLICAS=1 expose: - 80 - 8547 + - 9090 + - 9093 volumes: - admin:/admin @@ -70,6 +64,7 @@ services: links: - ystack-proxy:builds-registry.ystack.svc.cluster.local - ystack-proxy:buildkitd.ystack.svc.cluster.local + - ystack-proxy:monitoring.ystack.svc.cluster.local build: context: . # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore @@ -77,29 +72,48 @@ services: environment: - KUBECONFIG_WAIT=30 - KEEP_RUNNING=false + - EXAMPLES=sync-only sync-to-runtime in-cluster-build basic-dev-inner-loop + - CI=true volumes: - admin:/admin - ./examples:/usr/local/src/ystack/examples + - ./specs:/usr/local/src/ystack/specs entrypoint: - /bin/bash - -cx command: - | + [ "$$KEEP_RUNNING" = "true" ] || set -e + mkdir ~/.kube until test -f /admin/.kube/kubeconfig.yaml; do [ $$KUBECONFIG_WAIT -gt 0 ] || exit ${BULID_EXIT_CODE_ON_NO_CLUSTER:-0} KUBECONFIG_WAIT=$$(( $$KUBECONFIG_WAIT - 1 )) echo "Waiting for a kubeconfig ..." && sleep 1 done - set -e - cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config + + cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|master1|' > ~/.kube/config kubectl-waitretry --for=condition=Ready node --all - kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack + kubectl get nodes - y-cluster-install-prometheus-operator + echo "Our ystack-proxy image should install ystack essentials ..." + kubectl-waitretry --for=condition=Ready -n ystack pod -l app=minio + kubectl-waitretry --for=condition=Ready -n ystack pod -l ystack-builds-registry=http + kubectl-waitretry --for=condition=Ready -n ystack pod -l app=buildkitd + curl -f --retry 5 http://builds-registry.ystack.svc.cluster.local/v2/ + + y-cluster-assert-install --context=default + + for EXAMPLE in $$EXAMPLES; do + echo "# Running example $$EXAMPLE ..." + cd /usr/local/src/ystack/examples/$$EXAMPLE + y-skaffold run + done - cd /usr/local/src/ystack/examples/basic-dev-inner-loop - y-skaffold run + echo "# Running main specs ..." + kubectl create namespace ystack-specs + kubectl config set-context --current --namespace=ystack-specs + (cd /usr/local/src/ystack/specs && y-assert) if [ "$$KEEP_RUNNING" = "true" ]; then echo "Will stay running for manual work" diff --git a/example-knative-kafka/apiserver-source-kafka.yaml b/example-knative-kafka/apiserver-source-kafka.yaml deleted file mode 100644 index 36ffd787..00000000 --- a/example-knative-kafka/apiserver-source-kafka.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: events-sa ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: event-watcher -rules: - - apiGroups: - - "" - resources: - - events - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: k8s-ra-event-watcher -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: event-watcher -subjects: - - kind: ServiceAccount - name: events-sa - # can be reoved from rbac when we use Kustomize - namespace: test-knative-eventing ---- -apiVersion: sources.eventing.knative.dev/v1alpha1 -kind: ApiServerSource -metadata: - name: testevents -spec: - serviceAccountName: events-sa - mode: Resource - resources: - - apiVersion: v1 - kind: Event - sink: - apiVersion: eventing.knative.dev/v1alpha1 - kind: Broker - name: kafka001 diff --git a/example-knative-kafka/broker-trigger-kafka.yaml b/example-knative-kafka/broker-trigger-kafka.yaml deleted file mode 100644 index d093f109..00000000 --- a/example-knative-kafka/broker-trigger-kafka.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: eventing.knative.dev/v1alpha1 -kind: Broker -metadata: - name: kafka001 -spec: - channelTemplateSpec: - apiVersion: messaging.knative.dev/v1alpha1 - kind: KafkaChannel - spec: - numPartitions: 1 - replicationFactor: 1 ---- -apiVersion: sources.eventing.knative.dev/v1alpha1 -kind: CronJobSource -metadata: - name: every-second-minute-to-kafka001 -spec: - schedule: "*/2 * * * *" - data: '{"message": "Hello world!"}' - sink: - apiVersion: eventing.knative.dev/v1alpha1 - kind: Broker - name: kafka001 diff --git a/example-knative-kafka/inspect-kafka001.yaml b/example-knative-kafka/inspect-kafka001.yaml deleted file mode 100644 index aa585352..00000000 --- a/example-knative-kafka/inspect-kafka001.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: inspect-kafka001 - labels: &labels - app: inspect - category: eventing - type: broker - ref: kafka001 -spec: - replicas: 1 - selector: - matchLabels: *labels - template: - metadata: - labels: *labels - annotations: - kafka-bootstrap: bootstrap.kafka:9092 - spec: - containers: - - name: kafkacat - image: solsson/kafkacat:alpine@sha256:0d8cc772fea8399b17c6a62d7618a045ca5adc1b050eefdcc104a9c53b658b96 - env: - - name: BOOTSTRAP - valueFrom: - fieldRef: - fieldPath: metadata.annotations['kafka-bootstrap'] - - name: NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: TRIGGER_TO_INSPECT - valueFrom: - fieldRef: - fieldPath: metadata.labels['ref'] - args: - - -b - - $(BOOTSTRAP) - - -C - - -t - - knative-messaging-kafka.$(NAMESPACE).$(TRIGGER_TO_INSPECT)-kn-trigger - - -f - - |- - \nKey (%K bytes): %k - Value (%S bytes): %s - Timestamp: %T - Partition: %p - Offset: %o - Headers: %h\n - - -u diff --git a/example-knative-kafka/kustomization.yaml b/example-knative-kafka/kustomization.yaml deleted file mode 100644 index 9a62ac31..00000000 --- a/example-knative-kafka/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -namespace: test-knative-eventing -resources: -- sequence-kafka.yaml -# The sequence is independent, the remainder belongs together -- broker-trigger-kafka.yaml -- apiserver-source-kafka.yaml -- inspect-kafka001.yaml diff --git a/example-knative-kafka/sequence-kafka.yaml b/example-knative-kafka/sequence-kafka.yaml deleted file mode 100644 index 3002aa2a..00000000 --- a/example-knative-kafka/sequence-kafka.yaml +++ /dev/null @@ -1,93 +0,0 @@ -# https://github.com/knative/docs/blob/master/docs/eventing/samples/sequence/sequence-reply-to-event-display/README.md ---- -apiVersion: serving.knative.dev/v1beta1 -kind: Service -metadata: - name: first -spec: - template: - spec: - containers: - - image: us.gcr.io/probable-summer-223122/cmd-03315b715ae8f3e08e3a9378df706fbb@sha256:2656f39a7fcb6afd9fc79e7a4e215d14d651dc674f38020d1d18c6f04b220700 - env: - - name: STEP - value: "0" ---- -apiVersion: serving.knative.dev/v1beta1 -kind: Service -metadata: - name: second -spec: - template: - spec: - containers: - - image: us.gcr.io/probable-summer-223122/cmd-03315b715ae8f3e08e3a9378df706fbb@sha256:2656f39a7fcb6afd9fc79e7a4e215d14d651dc674f38020d1d18c6f04b220700 - env: - - name: STEP - value: "1" ---- -apiVersion: serving.knative.dev/v1beta1 -kind: Service -metadata: - name: third -spec: - template: - spec: - containers: - - image: us.gcr.io/probable-summer-223122/cmd-03315b715ae8f3e08e3a9378df706fbb@sha256:2656f39a7fcb6afd9fc79e7a4e215d14d651dc674f38020d1d18c6f04b220700 - env: - - name: STEP - value: "2" ---- -apiVersion: messaging.knative.dev/v1alpha1 -kind: Sequence -metadata: - name: sequence01 -spec: - # Broker uses spec - #channelTemplateSpec: - channelTemplate: - apiVersion: messaging.knative.dev/v1alpha1 - kind: KafkaChannel - spec: - numPartitions: 1 - replicationFactor: 1 - steps: - - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: first - - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: second - - ref: - apiVersion: serving.knative.dev/v1alpha1 - kind: Service - name: third - reply: - kind: Service - apiVersion: serving.knative.dev/v1alpha1 - name: event-display ---- -apiVersion: serving.knative.dev/v1beta1 -kind: Service -metadata: - name: event-display -spec: - template: - spec: - containers: - - image: gcr.io/knative-releases/github.com/knative/eventing-sources/cmd/event_display ---- -apiVersion: sources.eventing.knative.dev/v1alpha1 -kind: CronJobSource -metadata: - name: every-minute-to-sequence01 -spec: - schedule: "* * * * *" - data: '{"message": "Hello world!"}' - sink: - apiVersion: messaging.knative.dev/v1alpha1 - kind: Sequence - name: sequence01 diff --git a/examples/sync-to-runtime/index.js b/examples/sync-to-runtime/index.js new file mode 100644 index 00000000..7e7f01a3 --- /dev/null +++ b/examples/sync-to-runtime/index.js @@ -0,0 +1,16 @@ +// Unlike this example a typical runtime would contain the server and require the handler +// This runtime inefficiently re-requires the handler on every request + +// Must be the js file, not the folder, for require.cache deletion to work +const HANDLER = './my-handler/index.js'; + +const http = require('http'); +const server = http.createServer((req, res) => { + delete require.cache[require.resolve(HANDLER)]; + require(HANDLER)().then(response => { + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end(response); + }); +}); + +server.listen(8080, '0.0.0.0'); diff --git a/examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml b/examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml new file mode 100644 index 00000000..5a7ea08d --- /dev/null +++ b/examples/sync-to-runtime/k8s/sync-to-runtime-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: sync-to-runtime +spec: + selector: + matchLabels: + app: ystack-examples-sync-to-runtime + template: + metadata: + labels: + app: ystack-examples-sync-to-runtime + spec: + containers: + - name: runtime + image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-to-runtime + ports: + - containerPort: 8080 + command: + - node + - /usr/src/app diff --git a/examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml b/examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml new file mode 100644 index 00000000..32da6f07 --- /dev/null +++ b/examples/sync-to-runtime/k8s/sync-to-runtime-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: sync-to-runtime +spec: + ports: + - port: 8080 + selector: + app: ystack-examples-sync-to-runtime diff --git a/examples/sync-to-runtime/my-handler/index.js b/examples/sync-to-runtime/my-handler/index.js new file mode 100644 index 00000000..d46c56e3 --- /dev/null +++ b/examples/sync-to-runtime/my-handler/index.js @@ -0,0 +1,3 @@ +module.exports = async () => { + return "OK from handler"; +}; diff --git a/examples/sync-to-runtime/skaffold.yaml b/examples/sync-to-runtime/skaffold.yaml new file mode 100644 index 00000000..6da62321 --- /dev/null +++ b/examples/sync-to-runtime/skaffold.yaml @@ -0,0 +1,23 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: sync-to-runtime +build: + tagPolicy: + gitCommit: + variant: CommitSha + artifacts: + - image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-to-runtime + custom: + buildCommand: >- + y-build-like-sync + node:12.18.0-alpine3.11@sha256:53cd04d947b6c097fde5ea031622f917e8a61378890effd8d548b1204628a3fc + sync: + manual: + - src: '**' + dest: /usr/src/app +deploy: + kubectl: + manifests: + - k8s/sync-to-runtime-service.yaml + - k8s/sync-to-runtime-deployment.yaml diff --git a/k3s/10-minio/kustomization.yaml b/k3s/10-minio/kustomization.yaml new file mode 100644 index 00000000..a19d3220 --- /dev/null +++ b/k3s/10-minio/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: ystack +bases: +- ../../minio/standalone,defaultsecret diff --git a/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml b/k3s/20-builds-registry/builds-registry-magic-numbers.yaml similarity index 100% rename from k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml rename to k3s/20-builds-registry/builds-registry-magic-numbers.yaml diff --git a/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml b/k3s/20-builds-registry/builds-registry-replicas-1.yaml similarity index 100% rename from k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml rename to k3s/20-builds-registry/builds-registry-replicas-1.yaml diff --git a/k3s/10-builds-registry-service/kustomization.yaml b/k3s/20-builds-registry/kustomization.yaml similarity index 54% rename from k3s/10-builds-registry-service/kustomization.yaml rename to k3s/20-builds-registry/kustomization.yaml index f23bbc00..1f72dce7 100644 --- a/k3s/10-builds-registry-service/kustomization.yaml +++ b/k3s/20-builds-registry/kustomization.yaml @@ -1,4 +1,7 @@ +namespace: ystack bases: - ../../registry/builds-service +- ../../registry/generic,minio patchesStrategicMerge: - builds-registry-magic-numbers.yaml +- builds-registry-replicas-1.yaml diff --git a/k3s/30-y-assert-support/kustomization.yaml b/k3s/30-y-assert-support/kustomization.yaml new file mode 100644 index 00000000..aa648dd9 --- /dev/null +++ b/k3s/30-y-assert-support/kustomization.yaml @@ -0,0 +1,5 @@ +namespace: ystack +bases: +- ../../y-assert-support +resources: +- monitoring-nodeport-service.yaml diff --git a/k3s/30-y-assert-support/monitoring-nodeport-service.yaml b/k3s/30-y-assert-support/monitoring-nodeport-service.yaml new file mode 100644 index 00000000..cfd89975 --- /dev/null +++ b/k3s/30-y-assert-support/monitoring-nodeport-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: monitoring-nodeport +spec: + type: NodePort + selector: + app: monitoring-proxy + ports: + - name: prometheus + port: 9090 + protocol: TCP + targetPort: 9090 + - name: alertmanager + port: 9093 + protocol: TCP + targetPort: 9093 diff --git a/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml b/k3s/40-buildkit/buildkitd-nodeport-service.yaml similarity index 100% rename from k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml rename to k3s/40-buildkit/buildkitd-nodeport-service.yaml diff --git a/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml b/k3s/40-buildkit/buildkitd-replicas-0.yaml similarity index 100% rename from k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml rename to k3s/40-buildkit/buildkitd-replicas-0.yaml diff --git a/k3s/40-buildkit/kustomization.yaml b/k3s/40-buildkit/kustomization.yaml new file mode 100644 index 00000000..5d13bf1b --- /dev/null +++ b/k3s/40-buildkit/kustomization.yaml @@ -0,0 +1,8 @@ +namespace: ystack +bases: +- ../../buildkit +- ../../minio/standalone,defaultsecret +resources: +- buildkitd-nodeport-service.yaml +patchesStrategicMerge: +- buildkitd-replicas-0.yaml diff --git a/k3s/image/Dockerfile b/k3s/docker-image/Dockerfile similarity index 76% rename from k3s/image/Dockerfile rename to k3s/docker-image/Dockerfile index 8b8305f5..e1eb35bb 100644 --- a/k3s/image/Dockerfile +++ b/k3s/docker-image/Dockerfile @@ -1,6 +1,6 @@ -FROM rancher/k3s:v1.18.3-k3s1@sha256:faddccf57fa09d7e11698eae56cbcf4f764ffa5a6917de9f4326af0b26c9b746 +FROM rancher/k3s:v1.18.4-k3s1@sha256:2555ed1896512b320c175859a95a5f1d6a2c64b501a84bc45d853a9bd0be0dfd -COPY k3s/image/registries.yaml /etc/rancher/k3s/registries.yaml +COPY k3s/docker-image/registries.yaml /etc/rancher/k3s/registries.yaml # The clusterIP should be predefined RUN sed -i 's|http://builds-registry.ystack.svc.cluster.local|http://10.43.0.50|' /etc/rancher/k3s/registries.yaml diff --git a/k3s/image/registries.yaml b/k3s/docker-image/registries.yaml similarity index 100% rename from k3s/image/registries.yaml rename to k3s/docker-image/registries.yaml diff --git a/k3s/docker-ystack-proxy/Dockerfile b/k3s/docker-ystack-proxy/Dockerfile index 25d0022b..252800ff 100644 --- a/k3s/docker-ystack-proxy/Dockerfile +++ b/k3s/docker-ystack-proxy/Dockerfile @@ -3,12 +3,16 @@ FROM lachlanevenson/k8s-kubectl:v1.18.3@sha256:495f41912554fde10e41ac54e8df6fdbd RUN apk add --no-cache git COPY . /etc/ystack + +# Install the things we need to proxy, but nothing more because big apply chunks are hard to maintain RUN set -ex; \ mkdir -p /var/lib/rancher/k3s/server/manifests; \ for base in \ 00-ystack-namespace \ - 10-builds-registry-service \ - registry,minio,buildkit; do \ + 10-minio \ + 20-builds-registry \ + 30-y-assert-support \ + 40-buildkit; do \ kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \ done diff --git a/k3s/docker-ystack-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh index 50ae28e7..148f8e59 100755 --- a/k3s/docker-ystack-proxy/entrypoint.sh +++ b/k3s/docker-ystack-proxy/entrypoint.sh @@ -8,19 +8,31 @@ until test -f /admin/.kube/kubeconfig.yaml; do done set -e -cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config +cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|master1|' > ~/.kube/config kubectl-waitretry --for=condition=Ready node --all -kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/ +# Might speed up provision, due to the dependency minio -> registry -> builds, but should't be necessary +kubectl apply -f /var/lib/rancher/k3s/server/manifests/ystack-00-ystack-namespace.yaml +kubectl apply -f /var/lib/rancher/k3s/server/manifests/ystack-10-minio.yaml +kubectl-waitretry -n ystack --for=condition=Ready pod minio-0 -NODE=agent +kubectl apply -f /var/lib/rancher/k3s/server/manifests/ + +[ -z "$BUILDKITD_REPLICAS" ] || kubectl -n ystack scale --replicas=$BUILDKITD_REPLICAS statefulset/buildkitd + +NODE=master1 REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort}) BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort}) +# Assuming ordering is predictable ... +PROMETHEUS=$(kubectl -n ystack get service monitoring-nodeport -o jsonpath={.spec.ports[0].nodePort}) +ALERTMANAGER=$(kubectl -n ystack get service monitoring-nodeport -o jsonpath={.spec.ports[1].nodePort}) cat envoy.template.yaml \ | sed "s|{{ node }}|$NODE|g" \ | sed "s|{{ registry_nodeport }}|$REGISTRY|g" \ | sed "s|{{ buildkit_nodeport }}|$BUILDKIT|g" \ + | sed "s|{{ prometheus_nodeport }}|$PROMETHEUS|g" \ + | sed "s|{{ alertmanager_nodeport }}|$ALERTMANAGER|g" \ > /envoy.yaml # TODO do we pass on signals? diff --git a/k3s/docker-ystack-proxy/envoy.template.yaml b/k3s/docker-ystack-proxy/envoy.template.yaml index c34e7c8b..4bf7045e 100644 --- a/k3s/docker-ystack-proxy/envoy.template.yaml +++ b/k3s/docker-ystack-proxy/envoy.template.yaml @@ -33,6 +33,32 @@ static_resources: "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy stat_prefix: tcp_stats cluster: "buildkitd" + - name: prometheus + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9090 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "prometheus" + - name: alertmanager + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9093 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "alertmanager" clusters: - name: registry type: STRICT_DNS @@ -62,3 +88,31 @@ static_resources: socket_address: address: {{ node }} port_value: {{ buildkit_nodeport }} + - name: prometheus + type: STRICT_DNS + connect_timeout: 30s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_prometheus + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ prometheus_nodeport }} + - name: alertmanager + type: STRICT_DNS + connect_timeout: 30s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_alertmanager + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ alertmanager_nodeport }} diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/registry,minio,buildkit/kustomization.yaml deleted file mode 100644 index 4e47c13c..00000000 --- a/k3s/registry,minio,buildkit/kustomization.yaml +++ /dev/null @@ -1,10 +0,0 @@ -bases: -- ../../buildkit -- ../../minio/standalone,defaultsecret -- ../../registry/generic,minio -resources: -- buildkitd-nodeport-service.yaml -patchesStrategicMerge: -- builds-registry-replicas-1.yaml -# Making it work with ystack-proxy is a todo, and maybe we could do some kind of scale up on demand -- buildkitd-replicas-0.yaml diff --git a/runner.Dockerfile b/runner.Dockerfile index 1cf5741c..f00393f3 100644 --- a/runner.Dockerfile +++ b/runner.Dockerfile @@ -36,13 +36,13 @@ RUN y-buildctl COPY bin/y-container-structure-test /usr/local/src/ystack/bin/ RUN y-container-structure-test +COPY bin/y-crane /usr/local/src/ystack/bin/ +RUN y-crane + ENV SKAFFOLD_UPDATE_CHECK=false COPY bin/y-skaffold /usr/local/src/ystack/bin/ RUN y-skaffold -COPY --from=gcr.io/go-containerregistry/crane:57766c1fee7bdc67c4e70680efffed535d50b459 \ - /ko-app/crane /usr/local/bin/crane - COPY . /usr/local/src/ystack WORKDIR /usr/local/src/ystack diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore index cc62aead..87513e2f 100644 --- a/runner.Dockerfile.dockerignore +++ b/runner.Dockerfile.dockerignore @@ -1,9 +1,10 @@ * + +# ystack-runner components !bin/y-* bin/*-bin !bin/kubectl-* -#!metrics-server/ #!converge-generic/ !minio/ !registry/builds-service/ @@ -12,13 +13,17 @@ bin/*-bin !registry/node-update-hosts/ !registry/generic,minio/ !buildkit/ -#!monitoring/rbac-base/ -#!monitoring/instance-now/ +!y-assert-support/ # k3s configuration -!k3s/image/registries.yaml -!k3s/00-ystack-namespace -!k3s/10-builds-registry-service -!k3s/registry,minio,buildkit -!k3s/docker-ystack-proxy/envoy.template.yaml -!k3s/docker-ystack-proxy/entrypoint.sh +!k3s/docker-* +!k3s/0* +!k3s/1* +!k3s/2* +!k3s/3* +!k3s/4* +!k3s/5* +!k3s/6* +!k3s/7* +!k3s/8* +!k3s/9* diff --git a/specs/assert-assert.spec.js b/specs/assert-assert.spec.js new file mode 100644 index 00000000..9f872a09 --- /dev/null +++ b/specs/assert-assert.spec.js @@ -0,0 +1,11 @@ +// This is quite imporant to keep up-to-date while we're getting used to non-interactive Jest watch +const ASSERT_FILES_MIN = 4; + +describe("Ystack specs validity", () => { + + it(`Has seen at least ${ASSERT_FILES_MIN} spec files`, async () => { + expect(await promValue(`assert_files_seen{pod="${process.env.POD_NAME}"}`)) + .toBeGreaterThanOrEqual(ASSERT_FILES_MIN); + }); + +}); diff --git a/specs/kustomization.yaml b/specs/kustomization.yaml new file mode 100644 index 00000000..eacf2dae --- /dev/null +++ b/specs/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +bases: +- github.com/Yolean/kubernetes-assert/runtime-nodejs/kustomize?ref=41207ccf8536ce022919d9aef86d2124b2c142b1 +namePrefix: ystack- +commonLabels: + app: ystack +images: +- name: yolean/assert-runtime-nodejs + newName: builds-registry.ystack.svc.cluster.local/yolean/ystack-specs diff --git a/specs/minio.spec.js b/specs/minio.spec.js new file mode 100644 index 00000000..71659531 --- /dev/null +++ b/specs/minio.spec.js @@ -0,0 +1,12 @@ +describe("Ystack minio", () => { + + describe("http", () => { + + it("The blobs-minio service (a legacy name) uses port 80", async () => { + const blobs = await fetch('http://blobs-minio.ystack/').then(res => res.text()); + expect(blobs).toMatch(/AccessDenied/); + }); + + }); + +}); diff --git a/specs/monitoring.spec.js b/specs/monitoring.spec.js new file mode 100644 index 00000000..4792608b --- /dev/null +++ b/specs/monitoring.spec.js @@ -0,0 +1,33 @@ +describe("Monitoring", () => { + + describe("Ystack's proxy", () => { + + it("Has Prometheus on port 9090", async () => { + const prom = await fetch('http://monitoring.ystack:9090/api/v1/status/runtimeinfo').then(res => res.json()); + expect(prom).toHaveProperty('data.reloadConfigSuccess', true); + }); + + it("Has Alertmanager on port 9093", async () => { + const alerts = await fetch('http://monitoring.ystack:9093/api/v2/alerts').then(res => res.json()); + expect(alerts).toBeInstanceOf(Array); + }); + + it("Prometheus has the kubernetes-assert PodMonitor", async () => { + const config = await fetch('http://monitoring.ystack:9090/api/v1/status/config').then(res => res.json()); + expect(config).toHaveProperty('data.yaml'); + expect(config.data.yaml).toMatch(/job_name: monitoring\/kubernetes-assert\/0/); + }); + + it("Prometheus finds at lest one target (this one) for the PodMonitor", async () => { + const targets = await fetch('http://monitoring.ystack:9090/api/v1/targets?state=active').then(res => res.json()); + expect(targets).toHaveProperty('data.activeTargets'); + expect(targets.data.activeTargets).toEqual( + expect.arrayContaining([ + expect.objectContaining({scrapePool: 'monitoring/kubernetes-assert/0'}) + ]) + ); + }); + + }); + +}); diff --git a/specs/registry.spec.js b/specs/registry.spec.js new file mode 100644 index 00000000..c63c358d --- /dev/null +++ b/specs/registry.spec.js @@ -0,0 +1,8 @@ +describe("Builds registry", () => { + + it("Registry is up, with storage", async () => { + const v2 = await fetch('http://builds-registry.ystack/v2/').then(res => res.json()); + expect(v2).toEqual({}); + }); + +}); diff --git a/specs/skaffold.yaml b/specs/skaffold.yaml new file mode 100644 index 00000000..bfb67b9e --- /dev/null +++ b/specs/skaffold.yaml @@ -0,0 +1,23 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: ystack-specs +build: + tagPolicy: + gitCommit: + variant: CommitSha + artifacts: + - image: builds-registry.ystack.svc.cluster.local/yolean/ystack-specs + context: ./ + custom: + buildCommand: >- + y-build-like-sync + yolean/assert-runtime-nodejs:41207ccf8536ce022919d9aef86d2124b2c142b1@sha256:8c44b3817e81c4d6cb01b6decbffb7628b701a5b860161263b008f9d45646e35 + sync: + manual: + - src: '**' + dest: /usr/src/specs +deploy: + kustomize: {} +profiles: +- name: y-assert diff --git a/y-assert-support/kustomization.yaml b/y-assert-support/kustomization.yaml new file mode 100644 index 00000000..47b3d055 --- /dev/null +++ b/y-assert-support/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- monitoring-service.yaml +- monitoring-proxy-deployment.yaml diff --git a/y-assert-support/monitoring-proxy-deployment.yaml b/y-assert-support/monitoring-proxy-deployment.yaml new file mode 100644 index 00000000..2b4a3d6b --- /dev/null +++ b/y-assert-support/monitoring-proxy-deployment.yaml @@ -0,0 +1,160 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: monitoring-proxy + labels: + app: monitoring-proxy +spec: + replicas: 1 + selector: + matchLabels: + app: monitoring-proxy + template: + metadata: + labels: + app: monitoring-proxy + spec: + containers: + - name: proxy + image: envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89 + ports: + - containerPort: 9090 + - containerPort: 9093 + command: + - /usr/local/bin/envoy + - -l + - info + - --config-yaml + - | + admin: + access_log_path: /dev/null + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9901 + static_resources: + listeners: + - name: prometheus + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9090 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + stat_prefix: prometheus + access_log: + - name: envoy.file_access_log + filter: + not_health_check_filter: {} + typed_config: + "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + path: /dev/stdout + typed_json_format: + start_time: "%START_TIME%" + req_method: "%REQ(:METHOD)%" + req_path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + resp_code: "%RESPONSE_CODE%" + resp_flags: "%RESPONSE_FLAGS%" + bytes_recv: "%BYTES_RECEIVED%" + bytes_sent: "%BYTES_SENT%" + duration: "%DURATION%" + agent: "%REQ(USER-AGENT)%" + req_id: "%REQ(X-REQUEST-ID)%" + upstream_host: "%UPSTREAM_HOST%" + resp_upstream_service_time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%" + resp_redirect: "%RESP(LOCATION)%" + req_content_type: "%REQ(CONTENT-TYPE)%" + route_config: + name: prometheus + virtual_hosts: + - name: prometheus + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: prometheus + timeout: 30s + http_filters: + - name: envoy.filters.http.router + - name: alertmanager + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 9093 + filter_chains: + - filters: + - name: envoy.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager + stat_prefix: alertmanager + access_log: + - name: envoy.file_access_log + filter: + not_health_check_filter: {} + typed_config: + "@type": type.googleapis.com/envoy.config.accesslog.v2.FileAccessLog + path: /dev/stdout + typed_json_format: + start_time: "%START_TIME%" + req_method: "%REQ(:METHOD)%" + req_path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + resp_code: "%RESPONSE_CODE%" + resp_flags: "%RESPONSE_FLAGS%" + bytes_recv: "%BYTES_RECEIVED%" + bytes_sent: "%BYTES_SENT%" + duration: "%DURATION%" + agent: "%REQ(USER-AGENT)%" + req_id: "%REQ(X-REQUEST-ID)%" + upstream_host: "%UPSTREAM_HOST%" + resp_upstream_service_time: "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%" + resp_redirect: "%RESP(LOCATION)%" + req_content_type: "%REQ(CONTENT-TYPE)%" + route_config: + name: alertmanager + virtual_hosts: + - name: alertmanager + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: alertmanager + timeout: 30s + http_filters: + - name: envoy.filters.http.router + clusters: + - name: prometheus + type: STRICT_DNS + connect_timeout: 0.25s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: prometheus + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + protocol: TCP + port_value: 9090 + address: prometheus-now.monitoring.svc.cluster.local + - name: alertmanager + type: STRICT_DNS + connect_timeout: 0.25s + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: alertmanager + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + protocol: TCP + port_value: 9093 + address: alertmanager-main.monitoring.svc.cluster.local diff --git a/y-assert-support/monitoring-service.yaml b/y-assert-support/monitoring-service.yaml new file mode 100644 index 00000000..6e38a6c4 --- /dev/null +++ b/y-assert-support/monitoring-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: monitoring +spec: + selector: + app: monitoring-proxy + ports: + - name: prometheus + port: 9090 + protocol: TCP + targetPort: 9090 + - name: alertmanager + port: 9093 + protocol: TCP + targetPort: 9093