diff --git a/Jenkinsfile b/Jenkinsfile index 6e05c556fc..e96a994427 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -44,12 +44,12 @@ def genBuildStep(LinkedHashMap pkg, String arch) { sh 'docker info' } stage("build") { - try { - checkout scm - sh "make REF=$branch ${pkg.target}" - } finally { - sh "make clean" - } + checkout scm + sh "make clean" + sh "make REF=$branch ${pkg.target}" + } + stage("verify") { + sh "make IMAGE=${pkg.image} verify" } } } diff --git a/Makefile b/Makefile index 3d77c151cd..80cdae1386 100644 --- a/Makefile +++ b/Makefile @@ -97,3 +97,13 @@ static: checkout ## build static-compiled packages for p in $(DOCKER_BUILD_PKGS); do \ $(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) TARGETPLATFORM=$(TARGETPLATFORM) CONTAINERD_VERSION=$(CONTAINERD_VERSION) RUNC_VERSION=$(RUNC_VERSION) $${p}; \ done + +.PHONY: verify +verify: ## verify installation of packages +# to verify using packages from staging, use: make VERIFY_PACKAGE_REPO=stage IMAGE=ubuntu:focal verify + docker run $(VERIFY_PLATFORM) --rm -i \ + -v "$$(pwd):/v" \ + -e DEBIAN_FRONTEND=noninteractive \ + -e PACKAGE_REPO=$(VERIFY_PACKAGE_REPO) \ + -w /v \ + $(IMAGE) ./verify diff --git a/common.mk b/common.mk index 08890fbfc0..9ee597c2b0 100644 --- a/common.mk +++ b/common.mk @@ -43,6 +43,13 @@ DOCKER_SCAN_REF ?= v0.17.0 DOCKER_COMPOSE_REF ?= v2.5.1 DOCKER_BUILDX_REF ?= v0.8.2 +# Use "stage" to install dependencies from download-stage.docker.com during the +# verify step. Leave empty or use any other value to install from download.docker.com +VERIFY_PACKAGE_REPO ?= staging + +# Optional flags like --platform=linux/armhf +VERIFY_PLATFORM ?= + export BUILDTIME export DEFAULT_PRODUCT_LICENSE export PACKAGER_NAME diff --git a/install-containerd-helpers b/install-containerd-helpers new file mode 100644 index 0000000000..631ad57938 --- /dev/null +++ b/install-containerd-helpers @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +### +# Script Name: install-containerd-helpers +# +# Description: A library that containers helpers to install containerd on different +# distributions based on a package manager +### +set -x extglob + +# Steps taken from: https://docs.docker.com/install/linux/docker-ce/centos/ +function install_rpm_containerd() { + if [ "${PACKAGE_REPO}" = "stage" ]; then + REPO_URL="https://download-stage.docker.com/linux/${DIST_ID}/docker-ce-staging.repo" + else + REPO_URL="https://download.docker.com/linux/${DIST_ID}/docker-ce.repo" + fi + + # Install containerd dependency for non-zypper dependecies + echo "[DEBUG] Installing engine dependencies from ${REPO_URL}" + + # Note: we enable test channel to be able to test non-stable containerd packages as well. + # Once a containerd package becomes stable it will also be available in the test channel, + # so this logic works for both cases. + # (See also same logic in install_debian_containerd) + + if dnf --version; then + dnf config-manager --add-repo "${REPO_URL}" + dnf config-manager --set-disabled docker-ce-* + dnf config-manager --set-enabled docker-ce-test + dnf makecache + else + yum-config-manager --add-repo "${REPO_URL}" + yum-config-manager --disable docker-ce-* + yum-config-manager --enable docker-ce-test + yum makecache + fi +} + +# Steps taken from: https://docs.docker.com/install/linux/docker-ce/ubuntu/ +function install_debian_containerd() { + if [ "${PACKAGE_REPO}" = "stage" ]; then + REPO_URL="https://download-stage.docker.com/linux/${DIST_ID}" + else + REPO_URL="https://download.docker.com/linux/${DIST_ID}" + fi + + echo "[DEBUG] Installing engine dependencies from ${REPO_URL}" + + #TODO include this step in the get.docker.com installation script + # Make sure ca-certificates are up-to-date + update-ca-certificates -f + + curl -fsSL "${REPO_URL}/gpg" | apt-key add - + + if [ "${DIST_VERSION}" = "sid" ]; then + echo 'Debian sid ("unstable") cannot be used for packaging: replace with the actual codename' + exit 1 + fi + ARCH=$(dpkg --print-architecture) + + # Note: we enable test channel to be able to test non-stable containerd packages as well. + # Once a containerd package becomes stable it will also be available in the test channel, + # so this logic works for both cases. + # (See also same logic in install_rpm_containerd) + echo "deb [arch=${ARCH}] ${REPO_URL} ${DIST_VERSION} test" > /etc/apt/sources.list.d/docker.list + + apt-get update +} diff --git a/verify b/verify new file mode 100755 index 0000000000..e7b9b59f41 --- /dev/null +++ b/verify @@ -0,0 +1,142 @@ +#!/usr/bin/env bash + +### +# Script Name: verify +# +# Description: This runs a smoke test to verify that the packages can be installed corrected +### + +# build/${DIST_ID}/${DIST_VERSION}/${ARCH} - location of all packages +# Manually Testing: docker run --rm -it -v $(pwd):/v -w /v "centos:7" ./verify + +set -e + +source install-containerd-helpers + +function verify() { + if dpkg --version >/dev/null 2>/dev/null; then + verify_deb + elif rpm --version >/dev/null 2>/dev/null; then + verify_rpm + else + echo "[ERROR] Unable to determine base os:" + cat /etc/os-release + exit 1 + fi +} + +function verify_deb() { + # First install prerequisites for our script and dpkg and apt to run correctly. + # This list SHOULD NOT include dependencies of docker itself, otherwise we would + # not be able to verify that our packages specify all the required dependencies. + apt-get update + apt-get -y install --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + lsb-release \ + software-properties-common + + DIST_ID=$(source /etc/os-release; echo "$ID") + DIST_VERSION=$(lsb_release -sc) + if [ "${DIST_VERSION}" = "sid" ]; then + echo 'Debian sid ("unstable") cannot be used for packaging: replace with the actual codename' + exit 1 + fi + + install_debian_containerd + + packages=$(find "deb/debbuild/${DIST_ID}-${DIST_VERSION}/" -type f -name "*.deb") + # All local packages need to be prefixed with `./` or else apt-get doesn't understand where to pull from + packages=$(echo "${packages}" | awk '$0="./"$0' | xargs) + + ( + set -x + # Install the locally built packages using 'dpkg' because installing with + # 'apt-get' would attempt to install dependency packages (such as the CLI) + # from download.docker.com instead of the locally built CLI package. Given + # that 'dpkg -i' does not install any dependency (but will fail if depen- + # dencies are missing), we use the '--ignore-depends' option to ignore + # packages we know to be missing at this stage, and '--force-depends' to + # only warn about any other missing dependency. + # + # shellcheck disable=SC2086 + dpkg \ + --ignore-depends=containerd.io,iptables,libdevmapper,libdevmapper1.02.1 \ + --force-depends \ + -i ${packages} + + # After installing the local packages, we run 'apt-get install' with the + # '--fix-broken' option to trigger installation of the dependencies, which + # should succeed successfully. This step is to verify that not only the + # packages can be installed, but also that all dependencies (including + # containerd.io) can be resolved correctly for the distro that we built for, + # before going through the whole pipeline and publishing the packages. + # + # The '--no-upgrade' option is set to prevent apt from attempting to install + # packages from download(-stage).docker.com that we already installed using + # the local packages above. Without this, installing (e.g.) ./docker-ce-cli + # would result in apt installing "docker-ce" from the package repository and + # produce a "the following packages will be DOWNGRADED" error. + # + # shellcheck disable=SC2086 + apt-get -y install --no-install-recommends --no-upgrade --fix-broken ${packages} + ) + docker --version + # FIXME this tries to connect to the docker daemon, which isn't started + #if [ "$(uname -m)" = "x86_64" ]; then + # docker scan --accept-license --version; + #fi + dockerd --version + containerd --version + runc --version +} + +function verify_rpm() { + DIST_ID=$(. /etc/os-release; echo "${ID}") + DIST_VERSION=$(. /etc/os-release; echo "${VERSION_ID}" | cut -d'.' -f1) + + pkg_manager="yum" + pkg_config_manager="yum-config-manager" + if dnf --version; then + pkg_manager="dnf" + pkg_config_manager="dnf config-manager" + dnf clean all && dnf upgrade -y + ${pkg_manager} install -y 'dnf-command(config-manager)' + fi + + case ${DIST_ID}:${DIST_VERSION} in + ol:7*) + # Needed for container-selinux + ${pkg_config_manager} --enable ol7_addons + ;; + fedora*) + dnf install -y findutils + ;; + esac + + install_rpm_containerd + + # find all rpm packages, exclude src package + echo "[DEBUG] Installing engine rpms" + packages=$(find "rpm/rpmbuild/${DIST_ID}-${DIST_VERSION}/RPMS/" -type f -name "*.rpm" | sed '/src/d') + + # install all non-source packages + ( + set -x + ${pkg_manager} install -y ${packages} + ) + + docker --version + # FIXME this tries to connect to the docker daemon, which isn't started + #if [ "$(uname -m)" = "x86_64" ]; then + # docker scan --accept-license --version; + #fi + dockerd --version + containerd --version + runc --version + +} + +verify