[master] Makefile: add "verify" target to test install of packages

Jenkinsfile

@@ -44,12 +44,12 @@ def genBuildStep(LinkedHashMap pkg, String arch) {
                 sh 'docker info'
             }
             stage("build") {
-                try {
-                    checkout scm
-                    sh "make REF=$branch ${pkg.target}"
-                } finally {
-                    sh "make clean"
-                }
+                checkout scm
+                sh "make clean"
+                sh "make REF=$branch ${pkg.target}"
+            }
+            stage("verify") {
+                sh "make IMAGE=${pkg.image} verify"
             }
         }
     }

Makefile

@@ -97,3 +97,13 @@ static: checkout ## build static-compiled packages
 	for p in $(DOCKER_BUILD_PKGS); do \
 		$(MAKE) -C $@ VERSION=$(VERSION) GO_VERSION=$(GO_VERSION) TARGETPLATFORM=$(TARGETPLATFORM) CONTAINERD_VERSION=$(CONTAINERD_VERSION) RUNC_VERSION=$(RUNC_VERSION) $${p}; \
 	done
+
+.PHONY: verify
+verify: ## verify installation of packages
+# to verify using packages from staging, use: make VERIFY_PACKAGE_REPO=stage IMAGE=ubuntu:focal verify
+	docker run $(VERIFY_PLATFORM) --rm -i \
+		-v "$$(pwd):/v" \
+		-e DEBIAN_FRONTEND=noninteractive \
+		-e PACKAGE_REPO=$(VERIFY_PACKAGE_REPO) \
+		-w /v \
+		$(IMAGE) ./verify

common.mk

@@ -43,6 +43,13 @@ DOCKER_SCAN_REF    ?= v0.17.0
 DOCKER_COMPOSE_REF ?= v2.5.1
 DOCKER_BUILDX_REF  ?= v0.8.2
 
+# Use "stage" to install dependencies from download-stage.docker.com during the
+# verify step. Leave empty or use any other value to install from download.docker.com
+VERIFY_PACKAGE_REPO ?= staging
+
+# Optional flags like --platform=linux/armhf
+VERIFY_PLATFORM ?=
+
 export BUILDTIME
 export DEFAULT_PRODUCT_LICENSE
 export PACKAGER_NAME

install-containerd-helpers

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+###
+# Script Name:  install-containerd-helpers
+#
+# Description: A library that containers helpers to install containerd on different
+#              distributions based on a package manager
+###
+set -x extglob
+
+# Steps taken from: https://docs.docker.com/install/linux/docker-ce/centos/
+function install_rpm_containerd() {
+	if [ "${PACKAGE_REPO}" = "stage" ]; then
+		REPO_URL="https://download-stage.docker.com/linux/${DIST_ID}/docker-ce-staging.repo"
+	else
+		REPO_URL="https://download.docker.com/linux/${DIST_ID}/docker-ce.repo"
+	fi
+
+	# Install containerd dependency for non-zypper dependecies
+	echo "[DEBUG] Installing engine dependencies from ${REPO_URL}"
+
+	# Note: we enable test channel to be able to test non-stable containerd packages as well.
+	# Once a containerd package becomes stable it will also be available in the test channel,
+	# so this logic works for both cases.
+	# (See also same logic in install_debian_containerd)
+
+	if dnf --version; then
+		dnf config-manager --add-repo "${REPO_URL}"
+		dnf config-manager --set-disabled docker-ce-*
+		dnf config-manager --set-enabled docker-ce-test
+		dnf makecache
+	else
+		yum-config-manager --add-repo "${REPO_URL}"
+		yum-config-manager --disable docker-ce-*
+		yum-config-manager --enable docker-ce-test
+		yum makecache
+	fi
+}
+
+# Steps taken from: https://docs.docker.com/install/linux/docker-ce/ubuntu/
+function install_debian_containerd() {
+	if [ "${PACKAGE_REPO}" = "stage" ]; then
+		REPO_URL="https://download-stage.docker.com/linux/${DIST_ID}"
+	else
+		REPO_URL="https://download.docker.com/linux/${DIST_ID}"
+	fi
+
+	echo "[DEBUG] Installing engine dependencies from ${REPO_URL}"
+
+	#TODO include this step in the get.docker.com installation script
+	# Make sure ca-certificates are up-to-date
+	update-ca-certificates -f
+
+	curl -fsSL "${REPO_URL}/gpg" | apt-key add -
+
+	if [ "${DIST_VERSION}" = "sid" ]; then
+		echo 'Debian sid ("unstable") cannot be used for packaging: replace with the actual codename'
+		exit 1
+	fi
+	ARCH=$(dpkg --print-architecture)
+
+	# Note: we enable test channel to be able to test non-stable containerd packages as well.
+	# Once a containerd package becomes stable it will also be available in the test channel,
+	# so this logic works for both cases.
+	# (See also same logic in install_rpm_containerd)
+	echo "deb [arch=${ARCH}] ${REPO_URL} ${DIST_VERSION} test" > /etc/apt/sources.list.d/docker.list
+
+	apt-get update
+}

verify

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+
+###
+# Script Name:  verify
+#
+# Description: This runs a smoke test to verify that the packages can be installed corrected
+###
+
+# build/${DIST_ID}/${DIST_VERSION}/${ARCH} - location of all packages
+# Manually Testing: docker run --rm -it -v $(pwd):/v -w /v "centos:7" ./verify
+
+set -e
+
+source install-containerd-helpers
+
+function verify() {
+	if dpkg --version >/dev/null 2>/dev/null; then
+		verify_deb
+	elif rpm --version >/dev/null 2>/dev/null; then
+		verify_rpm
+	else
+		echo "[ERROR] Unable to determine base os:"
+		cat /etc/os-release
+		exit 1
+	fi
+}
+
+function verify_deb() {
+	# First install prerequisites for our script and dpkg and apt to run correctly.
+	# This list SHOULD NOT include dependencies of docker itself, otherwise we would
+	# not be able to verify that our packages specify all the required dependencies.
+	apt-get update
+	apt-get -y install --no-install-recommends \
+		apt-transport-https \
+		ca-certificates \
+		curl \
+		gnupg2 \
+		lsb-release \
+		software-properties-common
+
+	DIST_ID=$(source /etc/os-release; echo "$ID")
+	DIST_VERSION=$(lsb_release -sc)
+	if [ "${DIST_VERSION}" = "sid" ]; then
+		echo 'Debian sid ("unstable") cannot be used for packaging: replace with the actual codename'
+		exit 1
+	fi
+
+	install_debian_containerd
+
+	packages=$(find "deb/debbuild/${DIST_ID}-${DIST_VERSION}/" -type f -name "*.deb")
+	# All local packages need to be prefixed with `./` or else apt-get doesn't understand where to pull from
+	packages=$(echo "${packages}" | awk '$0="./"$0' | xargs)
+
+	(
+		set -x
+		# Install the locally built packages using 'dpkg' because installing with
+		# 'apt-get' would attempt to install dependency packages (such as the CLI)
+		# from download.docker.com instead of the locally built CLI package. Given
+		# that 'dpkg -i' does not install any dependency (but will fail if depen-
+		# dencies are missing), we use the '--ignore-depends' option to ignore
+		# packages we know to be missing at this stage, and '--force-depends' to
+		# only warn about any other missing dependency.
+		#
+		# shellcheck disable=SC2086
+		dpkg \
+			--ignore-depends=containerd.io,iptables,libdevmapper,libdevmapper1.02.1 \
+			--force-depends \
+			-i ${packages}
+
+		# After installing the local packages, we run 'apt-get install' with the
+		# '--fix-broken' option to trigger installation of the dependencies, which
+		# should succeed successfully. This step is to verify that not only the
+		# packages can be installed, but also that all dependencies (including
+		# containerd.io) can be resolved correctly for the distro that we built for,
+		# before going through the whole pipeline and publishing the packages.
+		#
+		# The '--no-upgrade' option is set to prevent apt from attempting to install
+		# packages from download(-stage).docker.com that we already installed using
+		# the local packages above. Without this, installing (e.g.) ./docker-ce-cli
+		# would result in apt installing "docker-ce" from the package repository and
+		# produce a "the following packages will be DOWNGRADED" error.
+		#
+		# shellcheck disable=SC2086
+		apt-get -y install --no-install-recommends --no-upgrade --fix-broken ${packages}
+	)
+	docker --version
+	# FIXME this tries to connect to the docker daemon, which isn't started
+	#if [ "$(uname -m)" = "x86_64" ]; then
+	#	docker scan --accept-license --version;
+	#fi
+	dockerd --version
+	containerd --version
+	runc --version
+}
+
+function verify_rpm() {
+	DIST_ID=$(. /etc/os-release; echo "${ID}")
+	DIST_VERSION=$(. /etc/os-release; echo "${VERSION_ID}" | cut -d'.' -f1)
+
+	pkg_manager="yum"
+	pkg_config_manager="yum-config-manager"
+	if dnf --version; then
+		pkg_manager="dnf"
+		pkg_config_manager="dnf config-manager"
+		dnf clean all && dnf upgrade -y
+		${pkg_manager} install -y 'dnf-command(config-manager)'
+	fi
+
+	case ${DIST_ID}:${DIST_VERSION} in
+	ol:7*)
+		# Needed for container-selinux
+		${pkg_config_manager} --enable ol7_addons
+		;;
+	fedora*)
+		dnf install -y findutils
+		;;
+	esac
+
+	install_rpm_containerd
+
+	# find all rpm packages, exclude src package
+	echo "[DEBUG] Installing engine rpms"
+	packages=$(find "rpm/rpmbuild/${DIST_ID}-${DIST_VERSION}/RPMS/" -type f -name "*.rpm" | sed '/src/d')
+
+	# install all non-source packages
+	(
+		set -x
+		${pkg_manager} install -y ${packages}
+	)
+
+	docker --version
+	# FIXME this tries to connect to the docker daemon, which isn't started
+	#if [ "$(uname -m)" = "x86_64" ]; then
+	#	docker scan --accept-license --version;
+	#fi
+	dockerd --version
+	containerd --version
+	runc --version
+
+}
+
+verify