diff --git a/bin/y-cluster-provision-gke b/bin/y-cluster-provision-gke index b6940e20..c6f86a37 100755 --- a/bin/y-cluster-provision-gke +++ b/bin/y-cluster-provision-gke @@ -4,6 +4,7 @@ set -e NAME="$1" [ -z "$NAME" ] && echo "First argument must be a cluster name" && exit 1 +shift 1 FLAGS="" @@ -18,13 +19,14 @@ gcloud container get-server-config --region=$REGION | grep "1.14" FLAGS="$FLAGS --cluster-version=1.14.6-gke.1" FLAGS="$FLAGS --image-type=UBUNTU_CONTAINERD" -FLAGS="$FLAGS --machine-type=g1-small" + FLAGS="$FLAGS --num-nodes=1" +FLAGS="$FLAGS --enable-autoscaling" +FLAGS="$FLAGS --max-nodes=10" # https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster#create_regional -#FLAGS="$FLAGS --region=$REGION" -# Temporary, non-regional -FLAGS="$FLAGS --zone=$REGION-a" +FLAGS="$FLAGS --region=$REGION" +# Use https://cloud.google.com/sdk/gcloud/reference/container/clusters/update#--node-locations to add zones, and notee that num-nodes is multiplied # https://cloud.google.com/kubernetes-engine/docs/how-to/small-cluster-tuning #FLAGS="$FLAGS --logging-service none --monitoring-service none" @@ -34,6 +36,7 @@ FLAGS="$FLAGS --no-enable-stackdriver-kubernetes" # https://cloud.google.com/kubernetes-engine/docs/how-to/node-auto-upgrades #FLAGS="$FLAGS --enable-autoupgrade" +# TODO maintenance window # https://cloud.google.com/kubernetes-engine/docs/how-to/container-native-load-balancing NETWORK=default @@ -79,7 +82,7 @@ echo "### What this script will try to do is ..." echo "Use the currently configured project: $PROJECT" echo "Use the currently configured region: $REGION" echo "Add authentication to KUBECONFIG=$KUBECONFIG" -echo "Create a GKE cluster using the following flags: $FLAGS" +echo "Create a GKE cluster using the following flags: $FLAGS $@" echo "" read -p 'Please confirm [y/n]? ' yn @@ -88,8 +91,7 @@ case $yn in * ) exit;; esac -#gcloud container clusters create "$NAME" $FLAGS -gcloud container clusters create "$NAME" --create-subnetwork "" $FLAGS +gcloud container clusters create "$NAME" $FLAGS "$@" GKE_CONTEXT_NAME=$(kubectl config current-context) @@ -105,27 +107,31 @@ echo gcloud compute routers nats create nat-config \ --auto-allocate-nat-external-ips echo "To clean up:" -#echo gcloud container clusters delete --region $REGION $NAME -echo gcloud container clusters delete --zone $REGION-a $NAME +echo gcloud container clusters delete --zone $REGION $NAME echo gcloud compute routers nats delete --router-region $REGION --router nat-router nat-config echo gcloud compute routers delete nat-router --region $REGION +kubectl create namespace ystack + +kubectl -n ystack apply -k $YSTACK_HOME/registry/node-update-containerd/ + ### To be moved to registry provision script, but we do want the initial context name +[ -z "${BUILDS_BUCKET_NAME}" ] && BUILDS_BUCKET_NAME=${GKE_CONTEXT_NAME}_builds + +kubectl create secret generic -n ystack registry-persistence --from-literal=gcsBucketName=${BUILDS_BUCKET_NAME} + gcloud compute backend-buckets list gsutil ls # https://cloud.google.com/config-connector/docs/overview -mkdir -p gke/registry -cat << EOF > gke/registry/builds-gcs-storagebucket.yaml +cat << EOF > $YSTACK_HOME/gke/registry/builds-gcs-storagebucket.yaml # https://github.com/GoogleCloudPlatform/k8s-config-connector/blob/master/resources/storagebucket/storage_v1alpha2_storagebucket.yaml apiVersion: storage.cnrm.cloud.google.com/v1alpha2 kind: StorageBucket metadata: - name: ${GKE_CONTEXT_NAME}_builds + name: ${BUILDS_BUCKET_NAME} spec: {} EOF -# TODO we also need to generate a bucket name, or a secret with a bucket name, for gke/registry/builds-gcs-bucket-storage.yaml - kubectl apply -f gke/registry/builds-gcs-storagebucket.yaml || \ - gsutil mb -c regional -l $REGION gs://${GKE_CONTEXT_NAME}_builds + gsutil mb -c regional -l $REGION gs://${BUILDS_BUCKET_NAME} diff --git a/converge-gke/kustomization.yaml b/converge-gke/kustomization.yaml new file mode 100644 index 00000000..67820f0d --- /dev/null +++ b/converge-gke/kustomization.yaml @@ -0,0 +1,10 @@ +namespace: ystack +bases: +- ../gke/storageclasses +- ../buildkit +- ../gke/registry +- ../registry/node-update-hosts +# Two ways to get nodes' containerd to allow pull from in-cluster .local service +# (assuming we already have DNS name lookup fixed through for example ../registry/hosts-update): +#- ../../registry/generic,tls/ # requires manual containerd restart until https://github.com/containerd/containerd/issues/3071 is resolved +- ../registry/node-update-containerd # also requires restart but needs no TLS support in registry so might be a simpler option until we have containerd#3071 diff --git a/examples/basic-dev-inner-loop/backend/Dockerfile b/examples/basic-dev-inner-loop/backend/Dockerfile index 63001ecb..19f539db 100644 --- a/examples/basic-dev-inner-loop/backend/Dockerfile +++ b/examples/basic-dev-inner-loop/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:10.15.3-alpine +FROM node:10.16.3-alpine@sha256:77c898d0da5e7bfb6e05c9a64de136ba4e03889a72f3c298e95df822a38f450d WORKDIR /app EXPOSE 3000 diff --git a/gke/registry/builds-gcs-bucket-storage.yaml b/gke/registry/builds-gcs-bucket-storage.yaml index 5d02355c..1badf5a2 100644 --- a/gke/registry/builds-gcs-bucket-storage.yaml +++ b/gke/registry/builds-gcs-bucket-storage.yaml @@ -7,9 +7,11 @@ spec: spec: containers: - name: docker-v2 - image: registry:2.7.1@sha256:a54bc9be148764891c44676ce8c44f1e53514c43b1bfbab87b896f4b9f0b5d99 env: - name: REGISTRY_STORAGE value: gcs - name: REGISTRY_STORAGE_GCS_BUCKET - value: # TODO must be generated by the provision script + valueFrom: + secretKeyRef: + name: registry-persistence + key: gcsBucketName diff --git a/gke/registry/kustomization.yaml b/gke/registry/kustomization.yaml index 034350cb..ddf39ee5 100644 --- a/gke/registry/kustomization.yaml +++ b/gke/registry/kustomization.yaml @@ -1,8 +1,4 @@ bases: - ../../registry/generic/ -# Two ways to get nodes' containerd to allow pull from in-cluster .local service -# (assuming we already have DNS name lookup fixed through for example ../registry/hosts-update): -#- ../../registry/generic,tls/ # requires manual containerd restart until https://github.com/containerd/containerd/issues/3071 is resolved -- ../../registry/node-update-containerd/ # also requires restart but needs no TLS support in registry so might be a simpler option until we have containerd#3071 patchesStrategicMerge: - builds-gcs-bucket-storage.yaml diff --git a/gke/storageclasses/default.yaml b/gke/storageclasses/default.yaml new file mode 100644 index 00000000..92bd9a93 --- /dev/null +++ b/gke/storageclasses/default.yaml @@ -0,0 +1,6 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: zonal-ssd-resizable-delete + annotations: + storageclass.kubernetes.io/is-default-class: "true" diff --git a/gke/storageclasses/kustomization.yaml b/gke/storageclasses/kustomization.yaml new file mode 100644 index 00000000..1143f2da --- /dev/null +++ b/gke/storageclasses/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- standard-non-default.yaml +- zonal-ssd-resizable-retain.yaml +- zonal-ssd-resizable-delete.yaml +patchesStrategicMerge: +- default.yaml diff --git a/gke/storageclasses/standard-non-default.yaml b/gke/storageclasses/standard-non-default.yaml new file mode 100644 index 00000000..33662f5b --- /dev/null +++ b/gke/storageclasses/standard-non-default.yaml @@ -0,0 +1,10 @@ +# based on a fresh GKE cluster's kubectl get storageclass standard -o yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: standard + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard diff --git a/gke/storageclasses/zonal-ssd-resizable-delete.yaml b/gke/storageclasses/zonal-ssd-resizable-delete.yaml new file mode 100644 index 00000000..40bfc221 --- /dev/null +++ b/gke/storageclasses/zonal-ssd-resizable-delete.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: zonal-ssd-resizable-delete +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Delete +allowVolumeExpansion: true +parameters: + type: pd-ssd diff --git a/gke/storageclasses/zonal-ssd-resizable-retain.yaml b/gke/storageclasses/zonal-ssd-resizable-retain.yaml new file mode 100644 index 00000000..7ea1a399 --- /dev/null +++ b/gke/storageclasses/zonal-ssd-resizable-retain.yaml @@ -0,0 +1,9 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: zonal-ssd-resizable-retain +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain +allowVolumeExpansion: true +parameters: + type: pd-ssd diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/node-update-containerd/daemonset.yaml index f2bcf551..294ac914 100644 --- a/registry/node-update-containerd/daemonset.yaml +++ b/registry/node-update-containerd/daemonset.yaml @@ -11,6 +11,8 @@ spec: labels: app: containerd-registry-http spec: + hostPID: true + hostNetwork: true initContainers: - name: update image: ubuntu:18.04@sha256:b88f8848e9a1a4e4558ba7cfc4acc5879e1d0e7ac06401409062ad2627e6fb58 @@ -18,19 +20,37 @@ spec: - name: etc-containerd mountPath: /etc/containerd readOnly: false + - name: results + mountPath: /results + env: + - name: CONFIG_PATH + value: /etc/containerd/config.toml + securityContext: + privileged: true command: - bash #to troubleshoot use: - -cex - -ce - | - cat << EOF >> /etc/containerd/config.toml - [plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"] - endpoint = ["http://builds-registry.ystack.svc.cluster.local"] - [plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"] - endpoint = ["http://prod-registry.ystack.svc.cluster.local"] - EOF - echo "TODO currently you must manually ssh to the node and run: systemctl restart containerd" - # maybe https://alexei-led.github.io/post/k8s_node_shell/ ? + if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then + echo '[plugins.cri.registry.mirrors."builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH + echo ' endpoint = ["http://builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH + touch /results/config-updated + fi + if ! grep prod-registry.ystack.svc.cluster.local $CONFIG_PATH; then + echo '[plugins.cri.registry.mirrors."prod-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH + echo ' endpoint = ["http://prod-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH + touch /results/config-updated + fi + cat $CONFIG_PATH + if [ -f /results/config-updated ]; then + echo "Configuration has been updated; restarting containerd (not expecting any output) ..." + nsenter --mount=/proc/1/ns/mnt -- systemctl restart containerd + echo "Restart triggered" + else + echo "No configration change" + fi + echo "Done" containers: - name: init-container-did-the-work image: gcr.io/google_containers/pause-amd64:3.1@sha256:59eec8837a4d942cc19a52b8c09ea75121acc38114a2c68b98983ce9356b8610 @@ -39,3 +59,5 @@ spec: - name: etc-containerd hostPath: path: /etc/containerd + - name: results + emptyDir: {} diff --git a/registry/node-update-hosts/daemonset.yaml b/registry/node-update-hosts/daemonset.yaml index 56404a13..6ad2e1e6 100644 --- a/registry/node-update-hosts/daemonset.yaml +++ b/registry/node-update-hosts/daemonset.yaml @@ -23,7 +23,6 @@ spec: value: >- builds-registry.ystack.svc.cluster.local prod-registry.ystack.svc.cluster.local - blobs-minio.ystack.svc.cluster.local command: - bash - -ce