Fix capz PR jobs - downloand and import image tars instead of retagging missing images (#552)

Signed-off-by: Mark Rossetti <marosset@microsoft.com>

Author: marosset

capz/templates/gmsa-pr.yaml

@@ -353,6 +353,8 @@ spec:
         set -o pipefail
         set -o errexit
 
+        CI_VERSION=${CI_VERSION}
+
         systemctl stop kubelet
         declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
         for BINARY in "$${BINARIES[@]}"; do
@@ -361,43 +363,27 @@ spec:
         done
         systemctl restart kubelet
 
-        # prepull images from gcr.io/k8s-staging-ci-images and retag it to
-        # registry.k8s.io so kubeadm can fetch correct images no matter what
+        # Download image tars from GCS, import into containerd, and retag
+        CI_DIR=/tmp/k8s-ci
+        mkdir -p $$CI_DIR
         declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
-        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+        CONTAINER_EXT="tar"
+        CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
         IMAGE_REGISTRY_PREFIX=registry.k8s.io
         for IMAGE in "$${IMAGES[@]}"; do
-          $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
-          $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
+          echo "* downloading package: $$CI_URL/$$IMAGE.$$CONTAINER_EXT"
+          curl --retry 10 --retry-delay 5 "$$CI_URL/$$IMAGE.$$CONTAINER_EXT" --output "$$CI_DIR/$$IMAGE.$$CONTAINER_EXT"
+          ctr -n k8s.io images import "$$CI_DIR/$$IMAGE.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
+          ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"$${CI_VERSION//+/_}"
+          ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"$${CI_VERSION//+/_}"
         done
 
         echo "kubeadm version: $(kubeadm version -o=short)"
-        echo "kubectl version: $(kubectl version --client=true --short=true)"
+        echo "kubectl version: $(kubectl version --client=true)"
         echo "kubelet version: $(kubelet --version)"
       owner: root:root
       path: /tmp/replace-k8s-binaries.sh
       permissions: "0744"
-    - content: |
-        #!/bin/bash
-
-        set -o nounset
-        set -o pipefail
-        set -o errexit
-
-        curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
-        tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
-        rm /tmp/yq_linux_amd64.tar.gz
-
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
-        systemctl stop kubelet
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
-        systemctl restart kubelet
-      owner: root:root
-      path: /tmp/replace-k8s-components.sh
-      permissions: "0744"
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
@@ -415,7 +401,6 @@ spec:
       - /var/lib/etcddisk
     postKubeadmCommands:
     - bash -c /tmp/node-log-query-kubelet-config.sh
-    - bash -c /tmp/replace-k8s-components.sh
     preKubeadmCommands:
     - bash -c /tmp/replace-k8s-binaries.sh
     useExperimentalRetryJoin: true

capz/templates/pr/patches/kubeadm-bootstrap-control-plane-pr.yaml

@@ -8,6 +8,8 @@
         set -o pipefail
         set -o errexit
 
+        CI_VERSION=${CI_VERSION}
+
         systemctl stop kubelet
         declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
         for BINARY in "$${BINARIES[@]}"; do
@@ -16,51 +18,28 @@
         done
         systemctl restart kubelet
 
-        # prepull images from gcr.io/k8s-staging-ci-images and retag it to
-        # registry.k8s.io so kubeadm can fetch correct images no matter what
+        # Download image tars from GCS, import into containerd, and retag
+        CI_DIR=/tmp/k8s-ci
+        mkdir -p $$CI_DIR
         declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
-        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+        CONTAINER_EXT="tar"
+        CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
         IMAGE_REGISTRY_PREFIX=registry.k8s.io
         for IMAGE in "$${IMAGES[@]}"; do
-          $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
-          $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
+          echo "* downloading package: $$CI_URL/$$IMAGE.$$CONTAINER_EXT"
+          curl --retry 10 --retry-delay 5 "$$CI_URL/$$IMAGE.$$CONTAINER_EXT" --output "$$CI_DIR/$$IMAGE.$$CONTAINER_EXT"
+          ctr -n k8s.io images import "$$CI_DIR/$$IMAGE.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
+          ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"$${CI_VERSION//+/_}"
+          ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"$${CI_VERSION//+/_}"
         done
 
         echo "kubeadm version: $(kubeadm version -o=short)"
-        echo "kubectl version: $(kubectl version --client=true --short=true)"
+        echo "kubectl version: $(kubectl version --client=true)"
         echo "kubelet version: $(kubelet --version)"
     path: /tmp/replace-k8s-binaries.sh
     owner: "root:root"
     permissions: "0744"
-- op: add
-  path: /spec/kubeadmConfigSpec/files/-
-  value:
-    content: |
-        #!/bin/bash
-
-        set -o nounset
-        set -o pipefail
-        set -o errexit
-
-        curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
-        tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
-        rm /tmp/yq_linux_amd64.tar.gz
-
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
-        systemctl stop kubelet
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
-        systemctl restart kubelet
-    path: /tmp/replace-k8s-components.sh
-    owner: "root:root"
-    permissions: "0744"
 - op: add
   path: /spec/kubeadmConfigSpec/preKubeadmCommands/-
   value:
     bash -c /tmp/replace-k8s-binaries.sh
-- op: add
-  path: /spec/kubeadmConfigSpec/postKubeadmCommands/-
-  value:
-    bash -c /tmp/replace-k8s-components.sh

capz/templates/windows-pr.yaml

@@ -353,6 +353,8 @@ spec:
         set -o pipefail
         set -o errexit
 
+        CI_VERSION=${CI_VERSION}
+
         systemctl stop kubelet
         declare -a BINARIES=("kubeadm" "kubectl" "kubelet")
         for BINARY in "$${BINARIES[@]}"; do
@@ -361,43 +363,27 @@ spec:
         done
         systemctl restart kubelet
 
-        # prepull images from gcr.io/k8s-staging-ci-images and retag it to
-        # registry.k8s.io so kubeadm can fetch correct images no matter what
+        # Download image tars from GCS, import into containerd, and retag
+        CI_DIR=/tmp/k8s-ci
+        mkdir -p $$CI_DIR
         declare -a IMAGES=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
-        [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+        CONTAINER_EXT="tar"
+        CI_URL="https://storage.googleapis.com/k8s-release-dev/ci/$${CI_VERSION}/bin/linux/amd64"
         IMAGE_REGISTRY_PREFIX=registry.k8s.io
         for IMAGE in "$${IMAGES[@]}"; do
-          $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"${CI_VERSION//+/_}"
-          $${SUDO} ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"${CI_VERSION//+/_}"
+          echo "* downloading package: $$CI_URL/$$IMAGE.$$CONTAINER_EXT"
+          curl --retry 10 --retry-delay 5 "$$CI_URL/$$IMAGE.$$CONTAINER_EXT" --output "$$CI_DIR/$$IMAGE.$$CONTAINER_EXT"
+          ctr -n k8s.io images import "$$CI_DIR/$$IMAGE.$$CONTAINER_EXT" || echo "* ignoring expected 'ctr images import' result"
+          ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"$${CI_VERSION//+/_}" $$IMAGE_REGISTRY_PREFIX/$$IMAGE:"$${CI_VERSION//+/_}"
+          ctr -n k8s.io images tag $$IMAGE_REGISTRY_PREFIX/$$IMAGE-amd64:"$${CI_VERSION//+/_}" gcr.io/k8s-staging-ci-images/$$IMAGE:"$${CI_VERSION//+/_}"
         done
 
         echo "kubeadm version: $(kubeadm version -o=short)"
-        echo "kubectl version: $(kubectl version --client=true --short=true)"
+        echo "kubectl version: $(kubectl version --client=true)"
         echo "kubelet version: $(kubelet --version)"
       owner: root:root
       path: /tmp/replace-k8s-binaries.sh
       permissions: "0744"
-    - content: |
-        #!/bin/bash
-
-        set -o nounset
-        set -o pipefail
-        set -o errexit
-
-        curl -L --retry 10 --retry-delay 5 https://github.com/mikefarah/yq/releases/download/v4.6.1/yq_linux_amd64.tar.gz --output /tmp/yq_linux_amd64.tar.gz
-        tar -xzvf /tmp/yq_linux_amd64.tar.gz -C /tmp && mv /tmp/yq_linux_amd64 /usr/bin/yq
-        rm /tmp/yq_linux_amd64.tar.gz
-
-        export KUBECONFIG=/etc/kubernetes/admin.conf
-        kubectl -n kube-system set image daemonset/kube-proxy kube-proxy="${REGISTRY}/kube-proxy:${KUBE_IMAGE_TAG}"
-        systemctl stop kubelet
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-apiserver:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-apiserver.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-controller-manager:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-controller-manager.yaml
-        yq e '.spec.containers[0].image = "${REGISTRY}/kube-scheduler:${KUBE_IMAGE_TAG}"' -i /etc/kubernetes/manifests/kube-scheduler.yaml
-        systemctl restart kubelet
-      owner: root:root
-      path: /tmp/replace-k8s-components.sh
-      permissions: "0744"
     initConfiguration:
       nodeRegistration:
         kubeletExtraArgs:
@@ -415,7 +401,6 @@ spec:
       - /var/lib/etcddisk
     postKubeadmCommands:
     - bash -c /tmp/node-log-query-kubelet-config.sh
-    - bash -c /tmp/replace-k8s-components.sh
     preKubeadmCommands:
     - bash -c /tmp/replace-k8s-binaries.sh
     useExperimentalRetryJoin: true