From 03c9737138ff684787c43248d69d0d8a1b5f36e6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 22 May 2020 21:00:19 +0200 Subject: [PATCH 01/15] Sets up a CI stack that can be used to validate y-build, almost: buildkitd needs to be proxied too --- README.md | 18 ++++++ converge-generic/kustomization.yaml | 3 + docker-compose.dev-overrides.yml | 18 ++++-- docker-compose.test.yml | 64 +++++++++++++++---- .../basic-dev-inner-loop/backend/Dockerfile | 2 +- k3s/00-ystack-namespace/kustomization.yaml | 2 + k3s/00-ystack-namespace/ystack-namespace.yaml | 4 ++ .../builds-registry-magic-numbers.yaml | 7 ++ .../kustomization.yaml | 4 ++ k3s/builds-registry-proxy/Dockerfile | 23 +++++++ k3s/builds-registry-proxy/entrypoint.sh | 25 ++++++++ k3s/builds-registry-proxy/envoy.template.yaml | 37 +++++++++++ k3s/image/Dockerfile | 13 +++- k3s/image/registries.yaml | 2 +- .../builds-registry-replicas-1.yaml | 6 ++ .../kustomization.yaml | 6 ++ .../builds-registry-service.yaml} | 0 registry/builds-service/kustomization.yaml | 2 + registry/generic,minio/kustomization.yaml | 1 - registry/generic/kustomization.yaml | 2 - registry/prod-service/kustomization.yaml | 2 + .../prod-registry-service-placeholder.yaml} | 0 runner.Dockerfile.dockerignore | 19 ++++-- test/.gitignore | 1 + 24 files changed, 231 insertions(+), 30 deletions(-) create mode 100644 k3s/00-ystack-namespace/kustomization.yaml create mode 100644 k3s/00-ystack-namespace/ystack-namespace.yaml create mode 100644 k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml create mode 100644 k3s/10-builds-registry-service/kustomization.yaml create mode 100644 k3s/builds-registry-proxy/Dockerfile create mode 100755 k3s/builds-registry-proxy/entrypoint.sh create mode 100644 k3s/builds-registry-proxy/envoy.template.yaml create mode 100644 k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml create mode 100644 k3s/registry,minio,buildkit/kustomization.yaml rename registry/{generic/service-builds.yaml => builds-service/builds-registry-service.yaml} (100%) create mode 100644 registry/builds-service/kustomization.yaml create mode 100644 registry/prod-service/kustomization.yaml rename registry/{generic/service-prod-placeholder.yaml => prod-service/prod-registry-service-placeholder.yaml} (100%) create mode 100644 test/.gitignore diff --git a/README.md b/README.md index 1474d661..168831fb 100644 --- a/README.md +++ b/README.md @@ -87,3 +87,21 @@ Still doesn't guard against mistakes, because `kubectl -n ystack-dev delete pod` Y-stack is opinionated on Kubernetes devops tooling as well. We therefore download some CLIs to the aforementioned `PATH` entry. + +## CI test suite + +``` +docker volume rm ystack_admin 2> /dev/null || true +./test.sh +``` + +## Development + +``` +compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrides.yml' +$compose down \ + ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true +sudo rm test/.kube/kubeconfig.yaml +$compose up --build -d builds-registry.ystack.svc.cluster.local +export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml +``` diff --git a/converge-generic/kustomization.yaml b/converge-generic/kustomization.yaml index 176fbad1..a34189d6 100644 --- a/converge-generic/kustomization.yaml +++ b/converge-generic/kustomization.yaml @@ -1,5 +1,8 @@ namespace: ystack bases: +- ../registry/builds-service +- ../registry/prod-service +- ../registry/node-update-hosts - ../buildkit - ../minio/standalone,defaultsecret #- ../registry/generic,kafka # Note that kafka isn't a base (as it has its own namespace) and must be applied separately diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml index 17666ca2..d625b646 100644 --- a/docker-compose.dev-overrides.yml +++ b/docker-compose.dev-overrides.yml @@ -3,15 +3,25 @@ services: server: ports: - - 6443:6443 + - 17143:17143 - node: + agent: ports: - - 80:30080 - - 443:30443 + # Failed to get traefik to work in docker-compose; using a predefined nodeport range instead + #- 80:80 + #- 443:443 + - "31710-31719:31710-31719" mem_limit: 8589934592 memswap_limit: 0 sut: environment: - KEEP_RUNNING=true + +volumes: + admin: + driver: local + driver_opts: + type: none + device: $PWD/test + o: bind diff --git a/docker-compose.test.yml b/docker-compose.test.yml index e66e65f2..576543c4 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -2,42 +2,72 @@ version: '2.4' services: server: - build: ./k3s/image + build: + context: . + dockerfile: ./k3s/image/Dockerfile command: - server + - --https-listen-port=17143 - --disable-agent - - -v=0 + - --node-name=server + - --no-deploy=traefik + - --no-deploy=servicelb + - --kube-apiserver-arg + - service-node-port-range=31710-31719 + tmpfs: + - /run + - /var/run + privileged: true environment: - - K3S_CLUSTER_SECRET=somethingtotallyrandom + - K3S_TOKEN=somethingtotallyrandom - K3S_KUBECONFIG_OUTPUT=/admin/.kube/kubeconfig.yaml - K3S_KUBECONFIG_MODE=666 + expose: + - 17143 + - 8472 + - 10250 volumes: - k3s-server:/var/lib/rancher/k3s - admin:/admin - # Host port not needed for CI - #ports: - #- 6443:6443 mem_limit: 300000000 memswap_limit: 0 - node: - build: ./k3s/image + agent: + depends_on: + - server + build: + context: . + dockerfile: ./k3s/image/Dockerfile tmpfs: - /run - /var/run privileged: true environment: - - K3S_URL=https://server:6443 - - K3S_CLUSTER_SECRET=somethingtotallyrandom - # Can also use K3S_TOKEN from /var/lib/rancher/k3s/server/node-token instead of K3S_CLUSTER_SECRET - #- K3S_TOKEN=K13849a67fc385fd3c0fa6133a8649d9e717b0258b3b09c87ffc33dae362c12d8c0::node:2e373dca319a0525745fd8b3d8120d9c + - K3S_URL=https://server:17143 + - K3S_TOKEN=somethingtotallyrandom + expose: + - 8472 + - 10250 mem_limit: 1500000000 memswap_limit: 0 - sut: + builds-registry.ystack.svc.cluster.local: depends_on: - server - - node + - agent + build: + context: . + dockerfile: ./k3s/builds-registry-proxy/Dockerfile + environment: + - KUBECONFIG_WAIT=30 + expose: + - 80 + volumes: + - admin:/admin + + sut: + depends_on: + - builds-registry.ystack.svc.cluster.local build: context: . # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore @@ -47,6 +77,7 @@ services: - KEEP_RUNNING=false volumes: - admin:/admin + - ./examples:/usr/local/src/ystack/examples entrypoint: - /bin/bash - -cx @@ -62,7 +93,12 @@ services: cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config kubectl-waitretry --for=condition=Ready node --all kubectl get namespace ystack 2>/dev/null || kubectl create namespace ystack + y-cluster-install-prometheus-operator + + cd /usr/local/src/ystack/examples/basic-dev-inner-loop + y-skaffold run + if [ "$$KEEP_RUNNING" = "true" ]; then echo "Will stay running for manual work" sleep infinity diff --git a/examples/basic-dev-inner-loop/backend/Dockerfile b/examples/basic-dev-inner-loop/backend/Dockerfile index 7cc39a1f..c195c42d 100644 --- a/examples/basic-dev-inner-loop/backend/Dockerfile +++ b/examples/basic-dev-inner-loop/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.16.3-alpine@sha256:77c898d0da5e7bfb6e05c9a64de136ba4e03889a72f3c298e95df822a38f450d +FROM node:12.16.3-alpine@sha256:84f5c9090cc741421b2376b2393e7d5fa2f89d36167c12eb57aef16dd64ecf6c ARG build_tag ARG build_timestamp diff --git a/k3s/00-ystack-namespace/kustomization.yaml b/k3s/00-ystack-namespace/kustomization.yaml new file mode 100644 index 00000000..6c95cac6 --- /dev/null +++ b/k3s/00-ystack-namespace/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- ystack-namespace.yaml diff --git a/k3s/00-ystack-namespace/ystack-namespace.yaml b/k3s/00-ystack-namespace/ystack-namespace.yaml new file mode 100644 index 00000000..08b42f64 --- /dev/null +++ b/k3s/00-ystack-namespace/ystack-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ystack diff --git a/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml b/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml new file mode 100644 index 00000000..4cbb91b5 --- /dev/null +++ b/k3s/10-builds-registry-service/builds-registry-magic-numbers.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Service +metadata: + name: builds-registry +spec: + type: NodePort + clusterIP: 10.43.0.50 diff --git a/k3s/10-builds-registry-service/kustomization.yaml b/k3s/10-builds-registry-service/kustomization.yaml new file mode 100644 index 00000000..f23bbc00 --- /dev/null +++ b/k3s/10-builds-registry-service/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../../registry/builds-service +patchesStrategicMerge: +- builds-registry-magic-numbers.yaml diff --git a/k3s/builds-registry-proxy/Dockerfile b/k3s/builds-registry-proxy/Dockerfile new file mode 100644 index 00000000..8763bf8f --- /dev/null +++ b/k3s/builds-registry-proxy/Dockerfile @@ -0,0 +1,23 @@ +FROM lachlanevenson/k8s-kubectl:v1.18.3@sha256:495f41912554fde10e41ac54e8df6fdbd0e859e893a4cab61c9224b296f3454d as kubectl + +COPY . /etc/ystack +RUN set -ex; \ + mkdir -p /var/lib/rancher/k3s/server/manifests; \ + for base in \ + 00-ystack-namespace \ + 10-builds-registry-service \ + registry,minio,buildkit; do \ + kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \ + done + +FROM envoyproxy/envoy:v1.14.1@sha256:ccaf9e0135bf498fb8396ad49defd7f8567bf706411d9a707efb3978fb842c89 + +COPY --from=kubectl /usr/local/bin/kubectl /usr/local/bin/kubectl + +COPY bin/kubectl-waitretry /usr/local/bin/kubectl-waitretry + +COPY --from=kubectl /var/lib/rancher/k3s/server/manifests /var/lib/rancher/k3s/server/manifests + +COPY k3s/builds-registry-proxy/envoy.template.yaml /envoy.template.yaml +COPY k3s/builds-registry-proxy/entrypoint.sh /usr/local/bin/entrypoint +ENTRYPOINT [ "entrypoint" ] diff --git a/k3s/builds-registry-proxy/entrypoint.sh b/k3s/builds-registry-proxy/entrypoint.sh new file mode 100755 index 00000000..9339ed26 --- /dev/null +++ b/k3s/builds-registry-proxy/entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +mkdir ~/.kube +until test -f /admin/.kube/kubeconfig.yaml; do + [ $KUBECONFIG_WAIT -gt 0 ] || exit 1 + KUBECONFIG_WAIT=$(( $KUBECONFIG_WAIT - 1 )) + echo "Waiting for a kubeconfig ..." && sleep 1 +done + +set -e +cat /admin/.kube/kubeconfig.yaml | sed 's|127.0.0.1|server|' > ~/.kube/config +kubectl-waitretry --for=condition=Ready node --all + +kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/ + +NODE=agent +NODEPORT=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort}) + +cat envoy.template.yaml \ + | sed "s|{{ node }}|$NODE|g" \ + | sed "s|{{ nodeport }}|$NODEPORT|g" \ + > /envoy.yaml + +# TODO do we pass on signals? +envoy --config-path /envoy.yaml diff --git a/k3s/builds-registry-proxy/envoy.template.yaml b/k3s/builds-registry-proxy/envoy.template.yaml new file mode 100644 index 00000000..9c47e158 --- /dev/null +++ b/k3s/builds-registry-proxy/envoy.template.yaml @@ -0,0 +1,37 @@ +admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 127.0.0.1 + port_value: 9903 +static_resources: + listeners: + - name: proxy + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "registry" + clusters: + - name: registry + type: STRICT_DNS + connect_timeout: 5s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_google + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ nodeport }} diff --git a/k3s/image/Dockerfile b/k3s/image/Dockerfile index 9b237e52..1cbfda01 100644 --- a/k3s/image/Dockerfile +++ b/k3s/image/Dockerfile @@ -1,3 +1,12 @@ -FROM rancher/k3s:v1.17.2-k3s1@sha256:674c68efbb2190284ee25c73c03fd9755fa77ad983e068b43542013a6bfe61de +FROM rancher/k3s:v1.17.5-k3s1@sha256:3025ecfa3c7c3a262a97435cb7c9c182d22b93d014bac767019d65bdce586786 -COPY registries.yaml /etc/rancher/k3s/registries.yaml +COPY k3s/image/registries.yaml /etc/rancher/k3s/registries.yaml + +# Unfortunately /var/lib/rancher/k3s is a VOLUME so this has no effect on server start +# COPY registry/generic /etc/ystack/registry/generic +# COPY k3s /etc/ystack/k3s +# RUN set -ex; \ +# mkdir -p /var/lib/rancher/k3s/server/manifests; \ +# for base in 00-ystack-namespace in-docker-builds-registry; do \ +# kubectl kustomize /etc/ystack/k3s/$base > /var/lib/rancher/k3s/server/manifests/ystack-$base.yaml; \ +# done diff --git a/k3s/image/registries.yaml b/k3s/image/registries.yaml index 423ef968..fe6889c9 100644 --- a/k3s/image/registries.yaml +++ b/k3s/image/registries.yaml @@ -1,7 +1,7 @@ mirrors: "builds-registry.ystack.svc.cluster.local": endpoint: - - http://builds-registry.ystack.svc.cluster.local + - http://10.43.0.50 "prod-registry.ystack.svc.cluster.local": endpoint: - http://prod-registry.ystack.svc.cluster.local diff --git a/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml b/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml new file mode 100644 index 00000000..21ec56b7 --- /dev/null +++ b/k3s/registry,minio,buildkit/builds-registry-replicas-1.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: registry +spec: + replicas: 1 diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/registry,minio,buildkit/kustomization.yaml new file mode 100644 index 00000000..aabf0e3a --- /dev/null +++ b/k3s/registry,minio,buildkit/kustomization.yaml @@ -0,0 +1,6 @@ +bases: +- ../../buildkit +- ../../minio/standalone,defaultsecret +- ../../registry/generic,minio +patchesStrategicMerge: +- builds-registry-replicas-1.yaml diff --git a/registry/generic/service-builds.yaml b/registry/builds-service/builds-registry-service.yaml similarity index 100% rename from registry/generic/service-builds.yaml rename to registry/builds-service/builds-registry-service.yaml diff --git a/registry/builds-service/kustomization.yaml b/registry/builds-service/kustomization.yaml new file mode 100644 index 00000000..92102610 --- /dev/null +++ b/registry/builds-service/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- builds-registry-service.yaml diff --git a/registry/generic,minio/kustomization.yaml b/registry/generic,minio/kustomization.yaml index 7e99d142..64881646 100644 --- a/registry/generic,minio/kustomization.yaml +++ b/registry/generic,minio/kustomization.yaml @@ -1,6 +1,5 @@ bases: - ../generic -- ../node-update-hosts resources: - bucket-create-ystack-builds.yaml patchesStrategicMerge: diff --git a/registry/generic/kustomization.yaml b/registry/generic/kustomization.yaml index 5be5a7d8..da0a800b 100644 --- a/registry/generic/kustomization.yaml +++ b/registry/generic/kustomization.yaml @@ -1,6 +1,4 @@ resources: -- service-builds.yaml -- service-prod-placeholder.yaml - deployment.yaml configMapGenerator: - name: registry-config diff --git a/registry/prod-service/kustomization.yaml b/registry/prod-service/kustomization.yaml new file mode 100644 index 00000000..d7f1a8c3 --- /dev/null +++ b/registry/prod-service/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- prod-registry-service-placeholder.yaml diff --git a/registry/generic/service-prod-placeholder.yaml b/registry/prod-service/prod-registry-service-placeholder.yaml similarity index 100% rename from registry/generic/service-prod-placeholder.yaml rename to registry/prod-service/prod-registry-service-placeholder.yaml diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore index 53910b47..6a9a34ef 100644 --- a/runner.Dockerfile.dockerignore +++ b/runner.Dockerfile.dockerignore @@ -3,13 +3,22 @@ bin/*-bin !bin/kubectl-* -# Y-stack currently contains a lot of experiments so let's maintain a (negated) list of what's actually kept up-to-date: -!metrics-server/ -!converge-generic/ +#!metrics-server/ +#!converge-generic/ !minio/ +!registry/builds-service/ +!registry/prod-service/ !registry/generic/ !registry/node-update-hosts/ !registry/generic,minio/ !buildkit/ -!monitoring/rbac-base/ -!monitoring/instance-now/ +#!monitoring/rbac-base/ +#!monitoring/instance-now/ + +# k3s configuration +!k3s/image/registries.yaml +!k3s/00-ystack-namespace +!k3s/10-builds-registry-service +!k3s/registry,minio,buildkit +!k3s/builds-registry-proxy/envoy.template.yaml +!k3s/builds-registry-proxy/entrypoint.sh diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..eb386402 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1 @@ +.kube From 28c6a927bce66f9480f799fc83c0abbf6f883aa5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 04:45:58 +0200 Subject: [PATCH 02/15] Tries to proxy buildkitd too, but skaffold run still fails Also fixes an issue with the buildkitd service's selector --- README.md | 2 +- docker-compose.test.yml | 10 +-- examples/basic-dev-inner-loop/skaffold.yaml | 2 +- k3s/builds-registry-proxy/envoy.template.yaml | 37 ----------- .../Dockerfile | 4 +- .../entrypoint.sh | 6 +- k3s/docker-ystack-proxy/envoy.template.yaml | 64 +++++++++++++++++++ k3s/image/Dockerfile | 2 + k3s/image/registries.yaml | 2 +- .../buildkitd-nodeport-service.yaml | 12 ++++ .../kustomization.yaml | 2 + runner.Dockerfile.dockerignore | 4 +- 12 files changed, 97 insertions(+), 50 deletions(-) delete mode 100644 k3s/builds-registry-proxy/envoy.template.yaml rename k3s/{builds-registry-proxy => docker-ystack-proxy}/Dockerfile (84%) rename k3s/{builds-registry-proxy => docker-ystack-proxy}/entrypoint.sh (69%) create mode 100644 k3s/docker-ystack-proxy/envoy.template.yaml create mode 100644 k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml diff --git a/README.md b/README.md index 168831fb..4242b59f 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,6 @@ compose='docker-compose -f docker-compose.test.yml -f docker-compose.dev-overrid $compose down \ ;docker volume rm ystack_admin ystack_k3s-server 2>/dev/null || true sudo rm test/.kube/kubeconfig.yaml -$compose up --build -d builds-registry.ystack.svc.cluster.local +$compose up --build -d ystack-proxy export KUBECONFIG=$PWD/test/.kube/kubeconfig.yaml ``` diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 576543c4..dfa958f6 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -51,23 +51,25 @@ services: mem_limit: 1500000000 memswap_limit: 0 - builds-registry.ystack.svc.cluster.local: + ystack-proxy: depends_on: - server - agent build: context: . - dockerfile: ./k3s/builds-registry-proxy/Dockerfile + dockerfile: ./k3s/docker-ystack-proxy/Dockerfile environment: - KUBECONFIG_WAIT=30 expose: - 80 + - 8547 volumes: - admin:/admin sut: - depends_on: - - builds-registry.ystack.svc.cluster.local + links: + - ystack-proxy:builds-registry.ystack.svc.cluster.local + - ystack-proxy:buildkitd.ystack.svc.cluster.local build: context: . # Note that with the current state of https://github.com/docker/docker-py/issues/2230 we must symlink the specific dockerignore to ./.dockerignore diff --git a/examples/basic-dev-inner-loop/skaffold.yaml b/examples/basic-dev-inner-loop/skaffold.yaml index 46a5ea42..40243f56 100644 --- a/examples/basic-dev-inner-loop/skaffold.yaml +++ b/examples/basic-dev-inner-loop/skaffold.yaml @@ -1,4 +1,4 @@ -apiVersion: skaffold/v2alpha2 +apiVersion: skaffold/v2beta4 kind: Config build: tagPolicy: diff --git a/k3s/builds-registry-proxy/envoy.template.yaml b/k3s/builds-registry-proxy/envoy.template.yaml deleted file mode 100644 index 9c47e158..00000000 --- a/k3s/builds-registry-proxy/envoy.template.yaml +++ /dev/null @@ -1,37 +0,0 @@ -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: - protocol: TCP - address: 127.0.0.1 - port_value: 9903 -static_resources: - listeners: - - name: proxy - address: - socket_address: - protocol: TCP - address: 0.0.0.0 - port_value: 80 - filter_chains: - - filters: - - name: tcp - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy - stat_prefix: tcp_stats - cluster: "registry" - clusters: - - name: registry - type: STRICT_DNS - connect_timeout: 5s - dns_lookup_family: V4_ONLY - lb_policy: ROUND_ROBIN - load_assignment: - cluster_name: service_google - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: {{ node }} - port_value: {{ nodeport }} diff --git a/k3s/builds-registry-proxy/Dockerfile b/k3s/docker-ystack-proxy/Dockerfile similarity index 84% rename from k3s/builds-registry-proxy/Dockerfile rename to k3s/docker-ystack-proxy/Dockerfile index 8763bf8f..80090525 100644 --- a/k3s/builds-registry-proxy/Dockerfile +++ b/k3s/docker-ystack-proxy/Dockerfile @@ -18,6 +18,6 @@ COPY bin/kubectl-waitretry /usr/local/bin/kubectl-waitretry COPY --from=kubectl /var/lib/rancher/k3s/server/manifests /var/lib/rancher/k3s/server/manifests -COPY k3s/builds-registry-proxy/envoy.template.yaml /envoy.template.yaml -COPY k3s/builds-registry-proxy/entrypoint.sh /usr/local/bin/entrypoint +COPY k3s/docker-ystack-proxy/envoy.template.yaml /envoy.template.yaml +COPY k3s/docker-ystack-proxy/entrypoint.sh /usr/local/bin/entrypoint ENTRYPOINT [ "entrypoint" ] diff --git a/k3s/builds-registry-proxy/entrypoint.sh b/k3s/docker-ystack-proxy/entrypoint.sh similarity index 69% rename from k3s/builds-registry-proxy/entrypoint.sh rename to k3s/docker-ystack-proxy/entrypoint.sh index 9339ed26..50ae28e7 100755 --- a/k3s/builds-registry-proxy/entrypoint.sh +++ b/k3s/docker-ystack-proxy/entrypoint.sh @@ -14,11 +14,13 @@ kubectl-waitretry --for=condition=Ready node --all kubectl -n ystack apply -f /var/lib/rancher/k3s/server/manifests/ NODE=agent -NODEPORT=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort}) +REGISTRY=$(kubectl -n ystack get service builds-registry -o jsonpath={.spec.ports[0].nodePort}) +BUILDKIT=$(kubectl -n ystack get service buildkitd-nodeport -o jsonpath={.spec.ports[0].nodePort}) cat envoy.template.yaml \ | sed "s|{{ node }}|$NODE|g" \ - | sed "s|{{ nodeport }}|$NODEPORT|g" \ + | sed "s|{{ registry_nodeport }}|$REGISTRY|g" \ + | sed "s|{{ buildkit_nodeport }}|$BUILDKIT|g" \ > /envoy.yaml # TODO do we pass on signals? diff --git a/k3s/docker-ystack-proxy/envoy.template.yaml b/k3s/docker-ystack-proxy/envoy.template.yaml new file mode 100644 index 00000000..c34e7c8b --- /dev/null +++ b/k3s/docker-ystack-proxy/envoy.template.yaml @@ -0,0 +1,64 @@ +admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + protocol: TCP + address: 127.0.0.1 + port_value: 9903 +static_resources: + listeners: + - name: registry + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 80 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "registry" + - name: buildkitd + address: + socket_address: + protocol: TCP + address: 0.0.0.0 + port_value: 8547 + filter_chains: + - filters: + - name: tcp + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + stat_prefix: tcp_stats + cluster: "buildkitd" + clusters: + - name: registry + type: STRICT_DNS + connect_timeout: 1s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_registry + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ registry_nodeport }} + - name: buildkitd + type: STRICT_DNS + connect_timeout: 30s + dns_lookup_family: V4_ONLY + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: service_buildkitd + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ node }} + port_value: {{ buildkit_nodeport }} diff --git a/k3s/image/Dockerfile b/k3s/image/Dockerfile index 1cbfda01..0b46d949 100644 --- a/k3s/image/Dockerfile +++ b/k3s/image/Dockerfile @@ -1,6 +1,8 @@ FROM rancher/k3s:v1.17.5-k3s1@sha256:3025ecfa3c7c3a262a97435cb7c9c182d22b93d014bac767019d65bdce586786 COPY k3s/image/registries.yaml /etc/rancher/k3s/registries.yaml +# The clusterIP should be predefined +RUN sed -i 's|http://builds-registry.ystack.svc.cluster.local|http://10.43.0.50|' /etc/rancher/k3s/registries.yaml # Unfortunately /var/lib/rancher/k3s is a VOLUME so this has no effect on server start # COPY registry/generic /etc/ystack/registry/generic diff --git a/k3s/image/registries.yaml b/k3s/image/registries.yaml index fe6889c9..423ef968 100644 --- a/k3s/image/registries.yaml +++ b/k3s/image/registries.yaml @@ -1,7 +1,7 @@ mirrors: "builds-registry.ystack.svc.cluster.local": endpoint: - - http://10.43.0.50 + - http://builds-registry.ystack.svc.cluster.local "prod-registry.ystack.svc.cluster.local": endpoint: - http://prod-registry.ystack.svc.cluster.local diff --git a/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml b/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml new file mode 100644 index 00000000..e6d36adc --- /dev/null +++ b/k3s/registry,minio,buildkit/buildkitd-nodeport-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: buildkitd-nodeport +spec: + type: NodePort + ports: + - port: 8547 + protocol: TCP + selector: + app: buildkitd + status: accepts-new-builds diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/registry,minio,buildkit/kustomization.yaml index aabf0e3a..f8e8e93b 100644 --- a/k3s/registry,minio,buildkit/kustomization.yaml +++ b/k3s/registry,minio,buildkit/kustomization.yaml @@ -2,5 +2,7 @@ bases: - ../../buildkit - ../../minio/standalone,defaultsecret - ../../registry/generic,minio +resources: +- buildkitd-nodeport-service.yaml patchesStrategicMerge: - builds-registry-replicas-1.yaml diff --git a/runner.Dockerfile.dockerignore b/runner.Dockerfile.dockerignore index 6a9a34ef..cc62aead 100644 --- a/runner.Dockerfile.dockerignore +++ b/runner.Dockerfile.dockerignore @@ -20,5 +20,5 @@ bin/*-bin !k3s/00-ystack-namespace !k3s/10-builds-registry-service !k3s/registry,minio,buildkit -!k3s/builds-registry-proxy/envoy.template.yaml -!k3s/builds-registry-proxy/entrypoint.sh +!k3s/docker-ystack-proxy/envoy.template.yaml +!k3s/docker-ystack-proxy/entrypoint.sh From 1b005f25f03db4638790cc91aeed91b9b7a355e7 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 05:20:49 +0200 Subject: [PATCH 03/15] Saves on resources when testing build without buildkit --- k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml | 6 ++++++ k3s/registry,minio,buildkit/kustomization.yaml | 2 ++ 2 files changed, 8 insertions(+) create mode 100644 k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml diff --git a/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml b/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml new file mode 100644 index 00000000..eaa1c7c6 --- /dev/null +++ b/k3s/registry,minio,buildkit/buildkitd-replicas-0.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: buildkitd +spec: + replicas: 0 diff --git a/k3s/registry,minio,buildkit/kustomization.yaml b/k3s/registry,minio,buildkit/kustomization.yaml index f8e8e93b..4e47c13c 100644 --- a/k3s/registry,minio,buildkit/kustomization.yaml +++ b/k3s/registry,minio,buildkit/kustomization.yaml @@ -6,3 +6,5 @@ resources: - buildkitd-nodeport-service.yaml patchesStrategicMerge: - builds-registry-replicas-1.yaml +# Making it work with ystack-proxy is a todo, and maybe we could do some kind of scale up on demand +- buildkitd-replicas-0.yaml From d86a1e32804a12af73278e12938a90ab575d29ee Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 05:08:31 +0200 Subject: [PATCH 04/15] First: skaffold init, then: skaffold run --port-forward=true --- examples/in-cluster-build/Dockerfile | 5 +++++ examples/in-cluster-build/index.html | 5 +++++ .../k8s/example-server-deployment.yaml | 18 ++++++++++++++++++ .../k8s/example-server-service.yaml | 9 +++++++++ examples/in-cluster-build/skaffold.yaml | 12 ++++++++++++ 5 files changed, 49 insertions(+) create mode 100644 examples/in-cluster-build/Dockerfile create mode 100644 examples/in-cluster-build/index.html create mode 100644 examples/in-cluster-build/k8s/example-server-deployment.yaml create mode 100644 examples/in-cluster-build/k8s/example-server-service.yaml create mode 100644 examples/in-cluster-build/skaffold.yaml diff --git a/examples/in-cluster-build/Dockerfile b/examples/in-cluster-build/Dockerfile new file mode 100644 index 00000000..72770a81 --- /dev/null +++ b/examples/in-cluster-build/Dockerfile @@ -0,0 +1,5 @@ +FROM busybox + +COPY index.html /var/www/ + +ENTRYPOINT [ "busybox", "httpd", "-f", "-p", "8080", "-h", "/var/www/" ] diff --git a/examples/in-cluster-build/index.html b/examples/in-cluster-build/index.html new file mode 100644 index 00000000..7008fda5 --- /dev/null +++ b/examples/in-cluster-build/index.html @@ -0,0 +1,5 @@ + + +Welcome + + diff --git a/examples/in-cluster-build/k8s/example-server-deployment.yaml b/examples/in-cluster-build/k8s/example-server-deployment.yaml new file mode 100644 index 00000000..b990d999 --- /dev/null +++ b/examples/in-cluster-build/k8s/example-server-deployment.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-server +spec: + selector: + matchLabels: + app: example-server + template: + metadata: + labels: + app: example-server + spec: + containers: + - name: busybox + image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server + ports: + - containerPort: 8080 diff --git a/examples/in-cluster-build/k8s/example-server-service.yaml b/examples/in-cluster-build/k8s/example-server-service.yaml new file mode 100644 index 00000000..1d7b5a7b --- /dev/null +++ b/examples/in-cluster-build/k8s/example-server-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: example-server +spec: + ports: + - port: 8080 + selector: + app: example-server diff --git a/examples/in-cluster-build/skaffold.yaml b/examples/in-cluster-build/skaffold.yaml new file mode 100644 index 00000000..059d50ad --- /dev/null +++ b/examples/in-cluster-build/skaffold.yaml @@ -0,0 +1,12 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: in-cluster-build +build: + artifacts: + - image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server +deploy: + kubectl: + manifests: + - k8s/example-server-deployment.yaml + - k8s/example-server-service.yaml From 0290337e1e35f51a978363ba485faa0f9c49ae73 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 05:20:24 +0200 Subject: [PATCH 05/15] Can also build with kaniku, which is unbearably slow --- examples/in-cluster-build/skaffold.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/in-cluster-build/skaffold.yaml b/examples/in-cluster-build/skaffold.yaml index 059d50ad..7819d8e8 100644 --- a/examples/in-cluster-build/skaffold.yaml +++ b/examples/in-cluster-build/skaffold.yaml @@ -5,6 +5,10 @@ metadata: build: artifacts: - image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server + kaniko: + reproducible: true + skipTLS: true + cluster: {} deploy: kubectl: manifests: From a6363b200d21a92b7ef8423526d4bd20bec2d35d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 05:28:06 +0200 Subject: [PATCH 06/15] Sync works nicely --- examples/in-cluster-build/index.html | 3 ++- examples/in-cluster-build/skaffold.yaml | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/in-cluster-build/index.html b/examples/in-cluster-build/index.html index 7008fda5..8fa09a01 100644 --- a/examples/in-cluster-build/index.html +++ b/examples/in-cluster-build/index.html @@ -1,5 +1,6 @@ -Welcome +With skaffold dev --port-forward=true you can edit this file locally +and just reload the page. diff --git a/examples/in-cluster-build/skaffold.yaml b/examples/in-cluster-build/skaffold.yaml index 7819d8e8..03140fa6 100644 --- a/examples/in-cluster-build/skaffold.yaml +++ b/examples/in-cluster-build/skaffold.yaml @@ -5,6 +5,8 @@ metadata: build: artifacts: - image: builds-registry.ystack.svc.cluster.local/ystack-examples/example-server + sync: + infer: ["**/*"] kaniko: reproducible: true skipTLS: true From d739de1e4a5de690cedfd012939ec42e75be3777 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 05:47:11 +0200 Subject: [PATCH 07/15] No, skaffold can't get the digest directly from kaniko --- docker-compose.dev-overrides.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker-compose.dev-overrides.yml b/docker-compose.dev-overrides.yml index d625b646..64d5a54a 100644 --- a/docker-compose.dev-overrides.yml +++ b/docker-compose.dev-overrides.yml @@ -14,6 +14,13 @@ services: mem_limit: 8589934592 memswap_limit: 0 + ystack-proxy: + ports: + # To run skaffold build/run/dev locally + - 80:80 + # To support y-build as skaffold build (TODO - proxying fails) + - 8547:8547 + sut: environment: - KEEP_RUNNING=true From a0ca6d1fe4de9103b1ed3420fc6cd92549238e57 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 23 May 2020 05:56:47 +0200 Subject: [PATCH 08/15] skipTLS seems to be skip TLS _verify_ which isn't desirable --- examples/in-cluster-build/skaffold.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/in-cluster-build/skaffold.yaml b/examples/in-cluster-build/skaffold.yaml index 03140fa6..8a6ad5a5 100644 --- a/examples/in-cluster-build/skaffold.yaml +++ b/examples/in-cluster-build/skaffold.yaml @@ -9,7 +9,6 @@ build: infer: ["**/*"] kaniko: reproducible: true - skipTLS: true cluster: {} deploy: kubectl: From cb5d3d05c638eb730bb9ee1e42da3220de1896c3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 25 May 2020 14:16:55 +0200 Subject: [PATCH 09/15] Reduces reliance on env propagation I don't think that kube access is used, unless we switch to buildctl's kubernetes protocol. --- bin/y-build | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/y-build b/bin/y-build index 8215ac29..4501c8d9 100755 --- a/bin/y-build +++ b/bin/y-build @@ -67,9 +67,9 @@ fi echo "Build command:" set -x -KUBECONFIG=$KUBECONFIG \ -BUILDKIT_HOST=$BUILDKIT_HOST \ - y-buildctl build \ +y-buildctl \ + --addr $BUILDKIT_HOST \ + build \ --frontend dockerfile.v0 \ --local context="$BUILD_CONTEXT" \ --local dockerfile="$BUILD_CONTEXT_DOCKERFILE" \ From 6257d2ef839f3d9f1c6990280eec459592329ab0 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 3 Jun 2020 14:14:33 +0200 Subject: [PATCH 10/15] Downgrade to k3s 1.16 because that's where GKE will be for three months --- bin/y-k3s-install | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/y-k3s-install b/bin/y-k3s-install index 56ae4a3e..bc5d9aac 100755 --- a/bin/y-k3s-install +++ b/bin/y-k3s-install @@ -21,8 +21,8 @@ mirrors: - http://prod-registry.ystack.svc.cluster.local EOF -INSTALLER_REVISION=58ebdb2a2ec5318ca40649eb7bd31679cb679f71 -export INSTALL_K3S_VERSION=v1.17.5+k3s1 +INSTALLER_REVISION=16c8554b5ca6636dbacaf5e00cee31dfea87e622 +export INSTALL_K3S_VERSION=v1.16.10+k3s1 curl -sfL https://github.com/rancher/k3s/raw/$INSTALLER_REVISION/install.sh | sh - service k3s start From ff8c56edab83ad1c989ff382b060d53a7f5e08eb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 3 Jun 2020 14:15:36 +0200 Subject: [PATCH 11/15] An idea for a NOP build, until Skaffold has a sync-only mode --- bin/y-build-nothing | 43 +++++++++++++++++++ .../sync-only/k8s/static-http-deployment.yaml | 40 +++++++++++++++++ examples/sync-only/skaffold.yaml | 22 ++++++++++ 3 files changed, 105 insertions(+) create mode 100755 bin/y-build-nothing create mode 100644 examples/sync-only/k8s/static-http-deployment.yaml create mode 100644 examples/sync-only/skaffold.yaml diff --git a/bin/y-build-nothing b/bin/y-build-nothing new file mode 100755 index 00000000..184fe2a7 --- /dev/null +++ b/bin/y-build-nothing @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +# Settings +DEFAULT_REGISTRY=builds-registry.ystack.svc.cluster.local +[ -z "$BUILDS_REGISTRY" ] && BUILDS_REGISTRY=$DEFAULT_REGISTRY +[ -z "$PUSH_REGISTRY" ] && PUSH_REGISTRY=$DEFAULT_REGISTRY + +if [ "$(curl -s --connect-timeout 3 http://$BUILDS_REGISTRY/v2/)" != "{}" ] +then + echo "ERROR Skaffold need local access to the builds registry for digest lookup" + echo "Registry: $BUILDS_REGISTRY" + echo "Look for y-stack's ingress or port-forward utilities" + exit 1 +fi + +[ -z "$IMAGE" ] && echo "No IMAGE env (from for example Skaffold)" && exit 1 + +IMAGE=$IMAGE +case "$IMAGE" in + $BUILDS_REGISTRY/* ) ;; + $PUSH_REGISTRY/* ) echo "Unlike y-build this script won't push to non-build registries" && exit 1 ;; + * ) echo "Output is restricted to PUSH_REGISTRY=$PUSH_REGISTRY. Got: $IMAGE" && exit 1 ;; +esac + +[ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx" + +CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9 +NOTHING_IMAGE=gcr.io/distroless/base:nonroot@sha256:54c459100e9d420e023b0aecc43f7010d2731b6163dd8e060906e2dec4c59890 + +BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") + +# TODO copy is significantly faster if the source image is already in the target repo + +# crane hangs for a long time if it doesn't know that the registry is plain http +NOTHING_IMAGE=$(echo $NOTHING_IMAGE | sed 's|.local|.local:80|') +IMAGE=$(echo $IMAGE | sed 's|.local|.local:80|') + +set -x +kubectl $ctx -n ystack run --restart=Never --attach \ + --image=$CRANE_IMAGE y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") -- \ + cp $NOTHING_IMAGE $IMAGE diff --git a/examples/sync-only/k8s/static-http-deployment.yaml b/examples/sync-only/k8s/static-http-deployment.yaml new file mode 100644 index 00000000..e025b304 --- /dev/null +++ b/examples/sync-only/k8s/static-http-deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: static-http +spec: + selector: + matchLabels: + app: ystack-examples-sync-only + template: + metadata: + labels: + app: ystack-examples-sync-only + spec: + containers: + - name: busybox + image: busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 + ports: + - containerPort: 8080 + command: + - busybox + - httpd + - -f + - -p + - "8080" + - -h + - /var/www/ + volumeMounts: + - name: www-static + mountPath: /var/www + - name: content + image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-only + command: + - sleep + - infinity + volumeMounts: + - name: www-static + mountPath: /var/www + volumes: + - name: www-static + emptyDir: {} diff --git a/examples/sync-only/skaffold.yaml b/examples/sync-only/skaffold.yaml new file mode 100644 index 00000000..7515adf2 --- /dev/null +++ b/examples/sync-only/skaffold.yaml @@ -0,0 +1,22 @@ +apiVersion: skaffold/v2beta4 +kind: Config +metadata: + name: sync-only +build: + tagPolicy: + gitCommit: + variant: CommitSha + artifacts: + - image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-only + custom: + buildCommand: y-build-nothing + sync: + manual: + - src: '**' + dest: /var/www + cluster: {} +deploy: + kubectl: + manifests: + - k8s/static-http-deployment.yaml + #- k8s/static-http-service.yaml From f5629d355201da07e2b593a7ff2b74531ab7dd72 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 3 Jun 2020 14:20:41 +0200 Subject: [PATCH 12/15] This starts, but file sync requires the tar command --- bin/y-build-nothing | 2 +- examples/sync-only/k8s/static-http-deployment.yaml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bin/y-build-nothing b/bin/y-build-nothing index 184fe2a7..69556395 100755 --- a/bin/y-build-nothing +++ b/bin/y-build-nothing @@ -27,7 +27,7 @@ esac [ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx" CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9 -NOTHING_IMAGE=gcr.io/distroless/base:nonroot@sha256:54c459100e9d420e023b0aecc43f7010d2731b6163dd8e060906e2dec4c59890 +NOTHING_IMAGE=gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610 BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") diff --git a/examples/sync-only/k8s/static-http-deployment.yaml b/examples/sync-only/k8s/static-http-deployment.yaml index e025b304..e78585cb 100644 --- a/examples/sync-only/k8s/static-http-deployment.yaml +++ b/examples/sync-only/k8s/static-http-deployment.yaml @@ -29,9 +29,6 @@ spec: mountPath: /var/www - name: content image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-only - command: - - sleep - - infinity volumeMounts: - name: www-static mountPath: /var/www From 0625bd80bcc516719fb1210ebda967e9c07286ce Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 4 Jun 2020 05:34:17 +0200 Subject: [PATCH 13/15] Busybox has tar and tail/sleep, but not nonroot or pause entrypoint --- bin/y-build-nothing | 2 +- examples/sync-only/index.html | 14 ++++++++++++++ examples/sync-only/k8s/static-http-deployment.yaml | 8 ++++++++ examples/sync-only/k8s/static-http-service.yaml | 9 +++++++++ examples/sync-only/skaffold.yaml | 2 +- 5 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 examples/sync-only/index.html create mode 100644 examples/sync-only/k8s/static-http-service.yaml diff --git a/bin/y-build-nothing b/bin/y-build-nothing index 69556395..2b87082e 100755 --- a/bin/y-build-nothing +++ b/bin/y-build-nothing @@ -27,7 +27,7 @@ esac [ -z "$ctx" ] && ctx="--context=$(kubectl config current-context)" && echo "No 'ctx' env, guessing: $ctx" CRANE_IMAGE=gcr.io/go-containerregistry/crane:96cf69f03a3cf2d9c8850ce9d8e1f33da354826a@sha256:7b4f1fc7c681e49ce7bf6ed95bd30d6c749f4a1cd2467f994fc045e6433ebde9 -NOTHING_IMAGE=gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610 +NOTHING_IMAGE=busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 BUILD_NAME=y-build-nothing-$(date -u +"%Y%m%dt%H%M%Sz") diff --git a/examples/sync-only/index.html b/examples/sync-only/index.html new file mode 100644 index 00000000..2444ff69 --- /dev/null +++ b/examples/sync-only/index.html @@ -0,0 +1,14 @@ + +Example sync + +This file will only be hosted (by the runtime container) after skaffold dev and modification. + +The example could be a bit lighter if: +
    +

    The NOTHING_IMAGE; was already in the target repo. Crane cp would be faster.

    +

    The image had a nonroot USER. We could remove securityContext.

    +

    The image had a sleep entrypoint. We could remove the command.

    +

    With https://github.com/kubernetes/kubernetes/issues/58512 we wouldn't need tar in the image.

    +
+ + diff --git a/examples/sync-only/k8s/static-http-deployment.yaml b/examples/sync-only/k8s/static-http-deployment.yaml index e78585cb..a67035f2 100644 --- a/examples/sync-only/k8s/static-http-deployment.yaml +++ b/examples/sync-only/k8s/static-http-deployment.yaml @@ -11,6 +11,10 @@ spec: labels: app: ystack-examples-sync-only spec: + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 containers: - name: busybox image: busybox@sha256:95cf004f559831017cdf4628aaf1bb30133677be8702a8c5f2994629f637a209 @@ -29,6 +33,10 @@ spec: mountPath: /var/www - name: content image: builds-registry.ystack.svc.cluster.local/ystack-examples/sync-only + command: + - tail + - -f + - /dev/null volumeMounts: - name: www-static mountPath: /var/www diff --git a/examples/sync-only/k8s/static-http-service.yaml b/examples/sync-only/k8s/static-http-service.yaml new file mode 100644 index 00000000..2d0785e4 --- /dev/null +++ b/examples/sync-only/k8s/static-http-service.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: static-http +spec: + ports: + - port: 8080 + selector: + app: ystack-examples-sync-only diff --git a/examples/sync-only/skaffold.yaml b/examples/sync-only/skaffold.yaml index 7515adf2..6cbb162c 100644 --- a/examples/sync-only/skaffold.yaml +++ b/examples/sync-only/skaffold.yaml @@ -18,5 +18,5 @@ build: deploy: kubectl: manifests: + - k8s/static-http-service.yaml - k8s/static-http-deployment.yaml - #- k8s/static-http-service.yaml From 772d2a9b373393c4a16938d48a5bac60c22e35fd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 5 Jun 2020 06:32:26 +0200 Subject: [PATCH 14/15] Draft runner for kubernetes-assert given a provisioned ystack cluster --- bin/y-assert | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 bin/y-assert diff --git a/bin/y-assert b/bin/y-assert new file mode 100755 index 00000000..c7f79526 --- /dev/null +++ b/bin/y-assert @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -e + +function log { + date -u +"%Y-%m-%dT%H:%M:%SZ" +} + +[ "$1" = "help" ] && echo ' +A convention for running https://github.com/Yolean/kubernetes-assert +using ephemeral namespaces. +' && exit 0 + +[ -z "$DIR" ] && DIR="$PWD" && echo "$(log) Module dir: $DIR" + +[ ! -f $DIR/skaffold.yaml ] && echo "Missing a skaffold.yaml in project dir $DIR" && exit 1 + +[ -z "$PROJECT" ] && PROJECT=$(echo $DIR | awk -F/ '{print $NF}' | tr -cd [:alnum:]- | tr '[:upper:]' '[:lower:]') + +[ -z "$ID" ] && ID=assert-$(date -u +"%Y%m%dt%H%M%Sz")-${PROJECT:0:10} && echo "$(log) Generated test ID: $ID" + +# What cleanup solutions are there? Let's start with https://github.com/hjacobs/kube-janitor +TTL="janitor/ttl: 23h" + +ctx=$1 +case $ctx in + "--context="*) shift 1 ;; + *) + [ "$CI" != "true" ] && echo "First arg should be --context" && exit 1 + ctx="--context=$(kubectl config current-context)" && echo "$(log) Current ctx: $ctx" + ;; +esac +skaffoldflags=$(echo $ctx | sed 's|^--|--kube-|') + +CIRUN=$1 +[ -z "$CIRUN" ] && CIRUN="run" + +# --- Up to this point we didn't actually do anything in any cluster, from here on we will --- + +if [ "$CI" = true ]; then + [ -f /run/secrets/kubernetes.io/serviceaccount/namespace ] && namespace=$(cat /run/secrets/kubernetes.io/serviceaccount/namespace) + [ -z "$namespace" ] && namespace=$(kubectl $ctx config view --minify --output 'jsonpath={..namespace}') + [ -z "$namespace" ] && echo "$(log) Failed to find a current namespace for CI=true mode" && exit 1 +else + namespace=$ID + echo "$(log) Creating (in $ctx) temp namespace: $namespace" + cat < Date: Fri, 5 Jun 2020 10:40:00 +0200 Subject: [PATCH 15/15] While y-k3s-install follows the GKE k8s minor maybe CI should run latest --- k3s/image/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k3s/image/Dockerfile b/k3s/image/Dockerfile index 0b46d949..8b8305f5 100644 --- a/k3s/image/Dockerfile +++ b/k3s/image/Dockerfile @@ -1,4 +1,4 @@ -FROM rancher/k3s:v1.17.5-k3s1@sha256:3025ecfa3c7c3a262a97435cb7c9c182d22b93d014bac767019d65bdce586786 +FROM rancher/k3s:v1.18.3-k3s1@sha256:faddccf57fa09d7e11698eae56cbcf4f764ffa5a6917de9f4326af0b26c9b746 COPY k3s/image/registries.yaml /etc/rancher/k3s/registries.yaml # The clusterIP should be predefined