diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ad3d14d003f1..738ee5c91f215 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -555,23 +555,6 @@ jobs: env sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report" - - name: Install tools - # The GitHub Actions image already has buildah and podman included. Not the - # ubuntu-24.04-arm images. buildah and podman is needed to convert the - # Kubernetes checkpoint archive to an OCI image. - # See contrib/checkpoint/checkpoint-restore-cri-test.sh - if: matrix.os == 'ubuntu-24.04-arm' - run: sudo apt-get install -y buildah podman - - - if: matrix.os != 'ubuntu-24.04-arm' - name: Checkpoint/Restore via CRI - env: - TEST_RUNTIME: ${{ matrix.runtime }} - CGROUP_DRIVER: ${{ matrix.cgroup_driver }} - run: | - env - sudo -E PATH=$PATH ./contrib/checkpoint/checkpoint-restore-cri-test.sh - - if: matrix.os != 'ubuntu-24.04-arm' name: Checkpoint/Restore Disable via CRI env: diff --git a/RELEASES.md b/RELEASES.md index d1af474243a99..bf50a03b65198 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -613,7 +613,7 @@ The deprecated features are shown in the following table: | Go-Plugin library (`*.so`) as containerd runtime plugin | containerd v2.0 | containerd v2.1 ✅ | Use external plugins (proxy or binary) | | NRI v0.1.0 plugin support | containerd v2.2 | containerd v2.3 | Use the v010-adapter NRI plugin, or update v0.1.0 plugins to use the current NRI API | | cgroup v1 support | containerd v2.2 | (May 2029) | Use cgroup v2 | -| Restoring checkpoint data during CRI `CreateContainer` | containerd v2.3 | containerd v2.4 | Follow [KEP-5823](https://github.com/kubernetes/enhancements/issues/5823) for a replacement `RestorePod` API | +| Restoring checkpoint data during CRI `CreateContainer` | containerd v2.3 | containerd v2.4 ✅ | Follow [KEP-5823](https://github.com/kubernetes/enhancements/issues/5823) for a replacement `RestorePod` API | - Pulling Schema 1 images has been disabled in containerd v2.0, but it still can be enabled by setting an environment variable `CONTAINERD_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE=1` until containerd v2.1. `ctr` users have to specify `--local` too (e.g., `ctr images pull --local`). Users of CRI clients (such as Kubernetes and `crictl`) have to specify this environment variable on the containerd daemon (usually in the systemd unit). diff --git a/contrib/checkpoint/checkpoint-restore-cri-disable-test.sh b/contrib/checkpoint/checkpoint-restore-cri-disable-test.sh index 1fd84f2cfe0ec..38af46d7f41ba 100755 --- a/contrib/checkpoint/checkpoint-restore-cri-disable-test.sh +++ b/contrib/checkpoint/checkpoint-restore-cri-disable-test.sh @@ -201,87 +201,33 @@ fi echo "PASS: Test 1: Checkpoint fails fast and source container remains running" cleanup_container "$pod_id" "$ctr_id" -# Test 2: Restore fails fast when enable_criu = false. -POD_JSON=$(mktemp) -jq ".log_directory=\"${TESTDIR}\"" "$TESTDATA"/sandbox_config.json >"$POD_JSON" -pod_id=$(crictl runp "$POD_JSON") -touch "$TESTDIR/dummy-checkpoint.tar" -RESTORE_JSON=$(mktemp) -jq ".image.image=\"$TESTDIR/dummy-checkpoint.tar\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON" -set +e -output=$(crictl create "$pod_id" "$RESTORE_JSON" "$POD_JSON" 2>&1) -exit_code=$? -set -e -rm -f "$RESTORE_JSON" "$POD_JSON" -if [ $exit_code -eq 0 ] || [[ ! "$output" =~ "criu support is disabled by configuration" ]]; then - echo "ERROR: Test 2 failed (restore did not fail fast). Output: $output" - exit 1 -fi -echo "PASS: Test 2: Restore fails fast when enable_criu = false" -crictl rmp -f "$pod_id" >/dev/null 2>&1 || true - -stop_containerd - -# ============================================================================== -# Group 2 (Configuration: enable_criu = true, normal PATH) -# ============================================================================== -start_containerd "true" "" - -# Test 3: Normal checkpoint and restore preserves container state. -ids=$(setup_container) -pod_id=$(echo "$ids" | cut -d: -f1) -ctr_id=$(echo "$ids" | cut -d: -f2) -crictl exec "$ctr_id" touch /root/state_file -rm -f "$TESTDIR"/state_checkpoint.tar -crictl checkpoint --export="$TESTDIR"/state_checkpoint.tar "${ctr_id}" -cleanup_container "$pod_id" "$ctr_id" - -POD_JSON=$(mktemp) -jq ".log_directory=\"${TESTDIR}\"" "$TESTDATA"/sandbox_config.json >"$POD_JSON" -pod_id=$(crictl runp "$POD_JSON") -RESTORE_JSON=$(mktemp) -jq ".image.image=\"$TESTDIR/state_checkpoint.tar\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON" -restored_ctr_id=$(crictl create "$pod_id" "$RESTORE_JSON" "$POD_JSON") -rm -f "$RESTORE_JSON" "$POD_JSON" -crictl start "$restored_ctr_id" -set +e -crictl exec "$restored_ctr_id" ls /root/state_file >/dev/null 2>&1 -exit_code=$? -set -e -if [ $exit_code -ne 0 ]; then - echo "ERROR: Test 3 failed (state_file not found in restored container)." - exit 1 -fi -echo "PASS: Test 3: Normal checkpoint and restore preserves container state" -cleanup_container "$pod_id" "$restored_ctr_id" - stop_containerd # ============================================================================== -# Group 3 (Configuration: enable_criu omitted/defaults, normal PATH) +# Group 2 (Configuration: enable_criu omitted/defaults, normal PATH) # ============================================================================== start_containerd "omit" "" -# Test 4: enable_criu omitted from configuration defaults to true (allowing checkpoint/restore). +# Test 2: enable_criu omitted from configuration defaults to true (allowing checkpoint). ids=$(setup_container) pod_id=$(echo "$ids" | cut -d: -f1) ctr_id=$(echo "$ids" | cut -d: -f2) rm -f "$TESTDIR"/omitted_checkpoint.tar crictl checkpoint --export="$TESTDIR"/omitted_checkpoint.tar "${ctr_id}" -echo "PASS: Test 4: enable_criu omitted from configuration defaults to true" +echo "PASS: Test 2: enable_criu omitted from configuration defaults to true" cleanup_container "$pod_id" "$ctr_id" stop_containerd # ============================================================================== -# Group 4 (Configuration: enable_criu = true, cleaned PATH without CRIU) +# Group 3 (Configuration: enable_criu = true, cleaned PATH without CRIU) # ============================================================================== if [ -n "${CRIU_DIR}" ]; then CLEANED_PATH=$(get_cleaned_path) start_containerd "true" "${CLEANED_PATH}" - # Test 5: CRIU missing from PATH, enable_criu = true. + # Test 3: CRIU missing from PATH, enable_criu = true. # Verifies that when CRIU is enabled but missing, it fails with the binary missing error. ids=$(setup_container) pod_id=$(echo "$ids" | cut -d: -f1) @@ -291,10 +237,10 @@ if [ -n "${CRIU_DIR}" ]; then exit_code=$? set -e if [ $exit_code -eq 0 ] || [[ ! "$output" =~ "criu binary not found" ]]; then - echo "ERROR: Test 5 failed. Output: $output" + echo "ERROR: Test 3 failed. Output: $output" exit 1 fi - echo "PASS: Test 5: Fails with binary not found as expected when enable_criu = true" + echo "PASS: Test 3: Fails with binary not found as expected when enable_criu = true" cleanup_container "$pod_id" "$ctr_id" stop_containerd diff --git a/contrib/checkpoint/checkpoint-restore-cri-test.sh b/contrib/checkpoint/checkpoint-restore-cri-test.sh deleted file mode 100755 index 427e5bd7a73f7..0000000000000 --- a/contrib/checkpoint/checkpoint-restore-cri-test.sh +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu -o pipefail - -DIR="$(dirname "${0}")" - -cd "${DIR}" -go build -o checkcriu - -if ! "./checkcriu"; then - echo >&2 "ERROR: CRIU check failed" - exit 1 -fi - -if [ ! -e "$(command -v crictl)" ]; then - echo >&2 "ERROR: crictl binary not found" - exit 1 -fi - -TESTDIR=$(mktemp -d -p "${PWD}") - -SUCCESS=0 - -function cleanup() { - rm -f ./checkcriu - crictl ps -a || true - pkill containerd || true - if [ "${SUCCESS}" == "1" ]; then - echo PASS - else - echo "--> containerd logs" - sed 's/^/----> \t/' <"${TESTDIR}/containerd.log" - echo FAIL - fi - umount "$(find "${TESTDIR}" -name shm -type d | head -1)" >/dev/null 2>&1 || true - umount "$(find "${TESTDIR}" -name rootfs -type d | head -1)" >/dev/null 2>&1 || true - rm -rf "${TESTDIR}" || true -} -trap cleanup EXIT - -TESTDATA=testdata -# shellcheck disable=SC2034 -export CONTAINERD_ADDRESS="$TESTDIR/c.sock" -export CONTAINER_RUNTIME_ENDPOINT="unix:///${CONTAINERD_ADDRESS}" - -# Generate crictl config file with 30s timeout -export CRI_CONFIG_FILE="${TESTDIR}/crictl.yaml" -cat < "${CRI_CONFIG_FILE}" -runtime-endpoint: unix://${CONTAINERD_ADDRESS} -image-endpoint: unix://${CONTAINERD_ADDRESS} -timeout: 30 -EOF - -TEST_IMAGE=ghcr.io/containerd/alpine - -function test_from_archive() { - echo "--> Deleting all pods: " - crictl -t 5s rmp -fa | sed 's/^/----> \t/' - echo -n "--> Pulling base image ${TEST_IMAGE}: " - crictl pull "${TEST_IMAGE}" - POD_JSON=$(mktemp) - # adapt the log directory - jq ".log_directory=\"${TESTDIR}\"" "$TESTDATA"/sandbox_config.json >"$POD_JSON" - echo -n "--> Start pod: " - pod_id=$(crictl runp "$POD_JSON") - echo "$pod_id" - CTR_JSON=$(mktemp) - jq '.annotations = {"cdi.k8s.io/device":"gpu","safe.annotation":"true"}' "$TESTDATA"/container_sleep.json >"$CTR_JSON" - echo -n "--> Create container: " - ctr_id=$(crictl create "$pod_id" "$CTR_JSON" "$POD_JSON") - echo "$ctr_id" - rm -f "$CTR_JSON" - echo -n "--> Start container: " - crictl start "$ctr_id" - lines_before=$(crictl logs "$ctr_id" | wc -l) - # changes file system to see if changes are included in the checkpoint - echo "--> Modifying container rootfs" - crictl exec "$ctr_id" touch /root/testfile - crictl exec "$ctr_id" rm /etc/motd - echo -n "--> Checkpointing container: " - crictl -t 10s checkpoint --export="$TESTDIR"/cp.tar "$ctr_id" - echo "--> Cleanup container: " - crictl rm -f "$ctr_id" | sed 's/^/----> \t/' - echo "--> Cleanup pod: " - crictl rmp -f "$pod_id" | sed 's/^/----> \t/' - echo "--> Cleanup images: " - crictl rmi "${TEST_IMAGE}" | sed 's/^/----> \t/' - echo -n "--> Start pod: " - pod_id=$(crictl runp "$POD_JSON") - echo "$pod_id" - # Replace original container with checkpoint image - RESTORE_JSON=$(mktemp) - jq ".image.image=\"$TESTDIR/cp.tar\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON" - echo -n "--> Create container from checkpoint: " - # This requires a larger timeout as we just deleted the image and - # pulling can take some time. - ctr_id=$(crictl -t 30s create "$pod_id" "$RESTORE_JSON" "$POD_JSON") - echo "$ctr_id" - rm -f "$RESTORE_JSON" "$POD_JSON" - echo -n "--> Start container from checkpoint: " - crictl start "$ctr_id" - sleep 1 - lines_after=$(crictl logs "$ctr_id" | wc -l) - if [ "$lines_before" -ge "$lines_after" ]; then - echo "number of lines after checkpointing ($lines_after) " \ - "should be larger than before checkpointing ($lines_before)" - false - fi - echo "--> Verifying CDI annotation filtering on restore: " - actual_annots=$(crictl inspect "$ctr_id" | jq -c '.status.annotations') - if jq -e 'has("cdi.k8s.io/device") or (has("safe.annotation") | not)' <<<"$actual_annots" >/dev/null; then - echo "error: CDI annotation was not filtered or safe annotation missing: $actual_annots" - exit 1 - fi - echo "--> Verifying deprecation warning via API (archive): " - archive_ts=$(../../bin/ctr deprecations list --format=json | jq -r '.[] | select(.id == "io.containerd.deprecation/cri-create-container-checkpoint-restore") | .lastOccurrence') - if [ -z "$archive_ts" ] || [ "$archive_ts" = "null" ]; then - echo "error: CRICreateContainerCheckpointRestore deprecation warning not found in API introspection (archive)" - exit 1 - fi - # Cleanup - echo "--> Cleanup images: " - (crictl rmi "${TEST_IMAGE}" || true) | sed 's/^/----> \t/' - echo -n "--> Verifying container rootfs: " - crictl exec "$ctr_id" ls -la /root/testfile - if crictl exec "$ctr_id" ls -la /etc/motd >/dev/null 2>&1; then - echo "error: file /etc/motd should not exist but it does" - exit 1 - fi - echo "--> Deleting all pods: " - crictl -t 5s rmp -fa | sed 's/^/----> \t/' - SUCCESS=1 -} - -function test_from_oci() { - echo "--> Deleting all pods: " - crictl -t 5s rmp -fa | sed 's/^/----> \t/' - echo -n "--> Pulling base image ${TEST_IMAGE}: " - crictl pull "${TEST_IMAGE}" - echo -n "--> Start pod: " - pod_id=$(crictl runp "$TESTDATA"/sandbox_config.json) - echo "$pod_id" - echo -n "--> Create container: " - ctr_id=$(crictl create "$pod_id" "$TESTDATA"/container_sleep.json "$TESTDATA"/sandbox_config.json) - echo "$ctr_id" - echo -n "--> Start container: " - crictl start "$ctr_id" - echo -n "--> Checkpointing container: " - crictl -t 10s checkpoint --export="$TESTDIR"/cp.tar "$ctr_id" - echo "--> Cleanup container: " - crictl rm -f "$ctr_id" | sed 's/^/----> \t/' - echo "--> Cleanup pod: " - crictl rmp -f "$pod_id" | sed 's/^/----> \t/' - echo "--> Cleanup pod: " - crictl rmi "${TEST_IMAGE}" | sed 's/^/----> \t/' - # Change cgroup of new sandbox - RESTORE_POD_JSON=$(mktemp) - jq ".linux.cgroup_parent=\"different_cgroup_789\"" "$TESTDATA"/sandbox_config.json >"$RESTORE_POD_JSON" - echo -n "--> Start pod: " - pod_id=$(crictl runp "$RESTORE_POD_JSON") - echo "$pod_id" - # Replace original container with checkpoint image - RESTORE_JSON=$(mktemp) - # Convert tar checkpoint archive to OCI image - echo "--> Convert checkpoint archive $TESTDIR/cp.tar to OCI image 'checkpoint-image:latest': " - newcontainer=$(buildah from scratch) - echo -n "----> Add checkpoint archive to new OCI image: " - buildah add "$newcontainer" "$TESTDIR"/cp.tar / - echo "----> Add checkpoint annotation to new OCI image: " - buildah config --annotation=org.criu.checkpoint.container.name=test "$newcontainer" - echo "----> Save new OCI image: " - buildah commit -q "$newcontainer" checkpoint-image:latest 2>&1 | sed 's/^/------> \t/' - echo "----> Cleanup temporary images: " - buildah rm "$newcontainer" | sed 's/^/------> \t/' - # Export OCI image to disk - echo "----> Export OCI image to disk: " - podman image save -q --format oci-archive -o "$TESTDIR"/oci.tar localhost/checkpoint-image:latest | sed 's/^/------> \t/' - echo "----> Cleanup temporary images: " - buildah rmi localhost/checkpoint-image:latest | sed 's/^/------> \t/' - # Remove potentially old version of the checkpoint image - echo "----> Cleanup potential old copies: " - ../../bin/ctr -n k8s.io images rm --sync localhost/checkpoint-image:latest 2>&1 | sed 's/^/------> \t/' - # Import image - echo "----> Import new image: " - ../../bin/ctr -n k8s.io images import "$TESTDIR"/oci.tar 2>&1 | sed 's/^/------> \t/' - jq ".image.image=\"localhost/checkpoint-image:latest\"" "$TESTDATA"/container_sleep.json >"$RESTORE_JSON" - echo -n "--> Create container from checkpoint: " - ctr_id=$(crictl -t 30s create "$pod_id" "$RESTORE_JSON" "$RESTORE_POD_JSON") - echo "$ctr_id" - rm -f "$RESTORE_JSON" "$RESTORE_POD_JSON" - echo -n "--> Start container from checkpoint: " - crictl start "$ctr_id" - echo "--> Verifying deprecation warning via API (oci): " - oci_ts=$(../../bin/ctr deprecations list --format=json | jq -r '.[] | select(.id == "io.containerd.deprecation/cri-create-container-checkpoint-restore") | .lastOccurrence') - if [ -z "$oci_ts" ] || [ "$oci_ts" = "null" ]; then - echo "error: CRICreateContainerCheckpointRestore deprecation warning not found in API introspection (oci)" - exit 1 - fi - if [ "$archive_ts" = "$oci_ts" ]; then - echo "error: expected lastOccurrence to update after OCI restore (was $archive_ts, now $oci_ts)" - exit 1 - fi - # Cleanup - echo "--> Cleanup images: " - ../../bin/ctr -n k8s.io images rm localhost/checkpoint-image:latest | sed 's/^/----> \t/' - echo "--> Cleanup images: " - (crictl rmi "${TEST_IMAGE}" || true) | sed 's/^/----> \t/' - echo "--> Deleting all pods: " - crictl -t 5s rmp -fa | sed 's/^/----> \t/' - SUCCESS=1 -} - -cat >"${TESTDIR}/config.toml" < Starting containerd: " -../../bin/containerd \ - --address "${TESTDIR}/c.sock" \ - --config "${TESTDIR}/config.toml" \ - --root "${TESTDIR}/root" \ - --state "${TESTDIR}/state" \ - --log-level trace &>"${TESTDIR}/containerd.log" & -# Make sure containerd is ready before calling critest. -retry_counter=0 -max_retries=10 -while true; do - ((retry_counter += 1)) - if crictl info 2>&1 | sed 's/^/----> \t/'; then - break - else - sleep 1.5 - fi - if [ "${retry_counter}" -gt "${max_retries}" ]; then - echo "--> Failed to start containerd" - exit 1 - fi - -done - -test_from_archive -SUCCESS=0 -test_from_oci diff --git a/contrib/checkpoint/checkpoint-restore-kubernetes-test.sh b/contrib/checkpoint/checkpoint-restore-kubernetes-test.sh deleted file mode 100755 index e81a6b3e0765f..0000000000000 --- a/contrib/checkpoint/checkpoint-restore-kubernetes-test.sh +++ /dev/null @@ -1,231 +0,0 @@ -#!/usr/bin/env bash - -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -eu -o pipefail - -DIR="$(dirname "${0}")" - -cd "${DIR}" -go build -o checkcriu - -if ! "./checkcriu"; then - echo >&2 "ERROR: CRIU check failed" - exit 1 -fi - -if [ ! -e "$(command -v crictl)" ]; then - echo >&2 "ERROR: crictl binary not found" - exit 1 -fi - -if [ ! -e "$(command -v kubectl)" ]; then - echo >&2 "ERROR: kubectl binary not found" - exit 1 -fi - -if [ ! -e "$(command -v ctr)" ]; then - echo >&2 "ERROR: ctr binary not found" - exit 1 -fi - -TESTDIR=$(mktemp -d) -OUTPUT=$(mktemp) -SUCCESS=0 - -function cleanup() { - # shellcheck disable=SC2317 - rm -f ./checkcriu - # shellcheck disable=SC2317 - rm -rf "${TESTDIR}" "${OUTPUT}" - # shellcheck disable=SC2317 - if [ "${SUCCESS}" == "1" ]; then - # shellcheck disable=SC2317 - echo PASS - else - # shellcheck disable=SC2317 - echo FAIL - fi -} -trap cleanup EXIT - -TESTDATA=testdata -export CONTAINER_RUNTIME_ENDPOINT="unix:///run/containerd/containerd.sock" -export KUBECONFIG=/var/run/kubernetes/admin.kubeconfig - -echo -n "--> Cleanup test pod/containers from previous run: " -kubectl delete pod sleeper --grace-period=1 || true -echo -n "--> Create new test pod/containers: " -kubectl apply -f $TESTDATA/sleep.yaml - -echo "--> Wait until test pod/containers are ready: " -while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].started}")" == "false" ]; do - echo "----> Waiting for pod/container sleeper/sleep to get ready" - sleep 0.5 -done - -echo "--> Do curl request to the test container: " -curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/' -COUNTER_BEFORE=$(cut -d\ -f2 <"${OUTPUT}") -echo "--> Check logs of test container: " -kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/' -LINES_BEFORE=$(wc -l <"${OUTPUT}") - -if [ ! -e /var/run/kubernetes/client-admin.crt ]; then - echo "Missing /var/run/kubernetes/client-admin.crt. Exiting" - exit 1 -fi - -if [ ! -e /var/run/kubernetes/client-admin.key ]; then - echo "Missing /var/run/kubernetes/client-admin.key. Exiting" - exit 1 -fi - -echo -n "--> Creating checkpoint: " -CP=$(curl -s --insecure --cert /var/run/kubernetes/client-admin.crt --key /var/run/kubernetes/client-admin.key -X POST "https://localhost:10250/checkpoint/default/sleeper/sleep" | jq -r ".items[0]") -echo "$CP" - -echo -n "--> Cleanup test pod/containers: " -kubectl delete pod sleeper --grace-period=1 - -OCI="localhost/checkpoint-image:latest" - -echo "--> Converting checkpoint archive $CP to OCI images $OCI" - -newcontainer=$(buildah from scratch) -echo -n "----> Add checkpoint archive to new OCI image: " -buildah add "$newcontainer" "$CP" / -echo "----> Add checkpoint annotation to new OCI image: " -buildah config --annotation=org.criu.checkpoint.container.name=test "$newcontainer" -echo "----> Save new OCI image: " -buildah commit -q "$newcontainer" "$OCI" 2>&1 | sed 's/^/------> \t/' -echo "----> Cleanup temporary images: " -buildah rm "$newcontainer" | sed 's/^/------> \t/' -# Export OCI image to disk -echo "----> Export OCI image to disk: " -podman image save -q --format oci-archive -o "$TESTDIR"/oci.tar "$OCI" | sed 's/^/------> \t/' -echo "----> Cleanup temporary images: " -buildah rmi "$OCI" | sed 's/^/------> \t/' -# Remove potentially old version of the checkpoint image -echo "----> Cleanup potential old copies: " -ctr -n k8s.io images rm --sync localhost/checkpoint-image:latest 2>&1 | sed 's/^/------> \t/' -# Import image -echo "----> Import checkpoint image: " -ctr -n k8s.io images import "$TESTDIR"/oci.tar 2>&1 | sed 's/^/------> \t/' - -echo -n "--> Create pod/containers from checkpoint: " -kubectl apply -f $TESTDATA/sleep-restore.yaml - -echo "--> Wait until test pod/containers are ready: " -while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].started}")" == "false" ]; do - echo "----> Waiting for pod/container sleeper/sleep to get ready" - sleep 0.5 -done - -echo "--> Do curl request to the test container: " -curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/' -COUNTER_AFTER=$(cut -d\ -f2 <"${OUTPUT}") -echo "--> Check logs of test container: " -kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/' -LINES_AFTER=$(wc -l <"${OUTPUT}") - -if [ "$LINES_BEFORE" -ge "$LINES_AFTER" ]; then - echo "number of lines after checkpointing ($LINES_AFTER) " \ - "should be larger than before checkpointing ($LINES_BEFORE)" - false -fi - -if [ "$COUNTER_BEFORE" -ge "$COUNTER_AFTER" ]; then - echo "number of lines after checkpointing ($COUNTER_AFTER) " \ - "should be larger than before checkpointing ($COUNTER_BEFORE)" - false -fi - -# Let's see if container restart also works correctly -CONTAINER_ID=$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].containerID}" | cut -d '/' -f 3) -CONTAINER_PID=$(crictl inspect "${CONTAINER_ID}" | jq .info.pid) - -echo "--> Kill container $CONTAINER_ID by killing PID $CONTAINER_PID" -# Kill PID in containner -kill -9 "${CONTAINER_PID}" - -# wait for the replacement container to come up -echo "--> Wait until replacement pod/containers is started: " -while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].restartCount}")" != "1" ]; do - echo "----> Waiting for pod/container sleeper/sleep to get ready" - sleep 1 -done - -echo "--> Do curl request to the test container: " -curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/' -COUNTER_AFTER=$(cut -d\ -f2 <"${OUTPUT}") -echo "--> Check logs of test container: " -kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/' -LINES_AFTER=$(wc -l <"${OUTPUT}") - -if [ "$LINES_BEFORE" -ge "$LINES_AFTER" ]; then - echo "number of lines after checkpointing ($LINES_AFTER) " \ - "should be larger than before checkpointing ($LINES_BEFORE)" - false -fi - -if [ "$COUNTER_BEFORE" -ge "$COUNTER_AFTER" ]; then - echo "number of lines after checkpointing ($COUNTER_AFTER) " \ - "should be larger than before checkpointing ($COUNTER_BEFORE)" - false -fi - -# Let's see if container restart also works correctly a second time - -CONTAINER_ID=$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].containerID}" | cut -d '/' -f 3) -CONTAINER_PID=$(crictl inspect "${CONTAINER_ID}" | jq .info.pid) - -# Kill PID in containner -echo "--> Kill container $CONTAINER_ID by killing PID $CONTAINER_PID" -kill -9 "${CONTAINER_PID}" - -# wait for the replacement container to come up -echo "--> Wait until replacement pod/containers is started: " -while [ "$(kubectl get pod sleeper -o jsonpath="{.status.containerStatuses[0].restartCount}")" != "2" ]; do - echo "----> Waiting for pod/container sleeper/sleep to get ready" - sleep 1 -done - -echo "--> Do curl request to the test container: " -curl -s "$(kubectl get pod sleeper --template '{{.status.podIP}}'):8088" | tee "${OUTPUT}" | sed 's/^/----> \t/' -COUNTER_AFTER=$(cut -d\ -f2 <"${OUTPUT}") -echo "--> Check logs of test container: " -kubectl logs sleeper -c sleep | tee "${OUTPUT}" | sed 's/^/----> \t/' -LINES_AFTER=$(wc -l <"${OUTPUT}") - -if [ "$LINES_BEFORE" -ge "$LINES_AFTER" ]; then - echo "number of lines after checkpointing ($LINES_AFTER) " \ - "should be larger than before checkpointing ($LINES_BEFORE)" - false -fi - -if [ "$COUNTER_BEFORE" -ge "$COUNTER_AFTER" ]; then - echo "number of lines after checkpointing ($COUNTER_AFTER) " \ - "should be larger than before checkpointing ($COUNTER_BEFORE)" - false -fi - -echo -n "--> Creating checkpoint from restored container: " -CP=$(curl -s --insecure --cert /var/run/kubernetes/client-admin.crt --key /var/run/kubernetes/client-admin.key -X POST "https://localhost:10250/checkpoint/default/sleeper/sleep" | jq -r ".items[0]") -echo "$CP" - -SUCCESS=1 - -exit 0 diff --git a/contrib/checkpoint/testdata/sleep-restore.yaml b/contrib/checkpoint/testdata/sleep-restore.yaml deleted file mode 100644 index 79937f399fa2e..0000000000000 --- a/contrib/checkpoint/testdata/sleep-restore.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: sleeper -spec: - containers: - - name: sleep - image: localhost/checkpoint-image:latest - imagePullPolicy: IfNotPresent - restartPolicy: Always diff --git a/integration/client/container_linux_test.go b/integration/client/container_linux_test.go index 1de8776cc9ac7..6a44e62e0732c 100644 --- a/integration/client/container_linux_test.go +++ b/integration/client/container_linux_test.go @@ -19,6 +19,7 @@ package client import ( "bytes" "context" + "encoding/json" "fmt" "io" "os" @@ -40,15 +41,20 @@ import ( . "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/core/containers" + coreimages "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/integration/failpoint" "github.com/containerd/containerd/v2/integration/images" + "github.com/containerd/containerd/v2/pkg/archive" "github.com/containerd/containerd/v2/pkg/cio" "github.com/containerd/containerd/v2/pkg/fifosync" "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/containerd/v2/pkg/shim" "github.com/containerd/containerd/v2/pkg/sys" + "github.com/containerd/containerd/v2/pkg/testutil" "github.com/containerd/containerd/v2/plugins" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" "github.com/stretchr/testify/require" "golang.org/x/sync/semaphore" @@ -1815,6 +1821,79 @@ func TestIssue10589(t *testing.T) { assert.Equal(t, Stopped, status.Status) } +// buildWhiteoutImage builds an image on top of the base image, appending +// layers generated with archive.WriteDiff that create files and then delete +// them again with whiteouts: +// +// touch /file-to-delete +// rm /file-to-delete +// mkdir /dir-to-delete && touch /dir-to-delete/foo +// rm -rf /dir-to-delete +// +// The image is returned as the blobs of the new layers, config, and manifest +// keyed by digest, together with the manifest descriptor, to be served with +// testutil.ServeImage. The blobs of the base image are not included; they +// are expected to be present in the content store already. +func buildWhiteoutImage(ctx context.Context, t *testing.T, client *Client, base Image) (map[digest.Digest][]byte, ocispec.Descriptor) { + blobs := map[digest.Digest][]byte{} + + empty := t.TempDir() + withFile := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(withFile, "file-to-delete"), nil, 0644)) + withDir := t.TempDir() + require.NoError(t, os.Mkdir(filepath.Join(withDir, "dir-to-delete"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(withDir, "dir-to-delete", "foo"), nil, 0644)) + + config, err := base.Spec(ctx) + require.NoError(t, err) + manifest, err := coreimages.Manifest(ctx, client.ContentStore(), base.Target(), platforms.Default()) + require.NoError(t, err) + + for _, l := range []struct { + createdBy string + lower, upper string + }{ + {"touch /file-to-delete", empty, withFile}, + {"rm /file-to-delete", withFile, empty}, + {"mkdir /dir-to-delete && touch /dir-to-delete/foo", empty, withDir}, + {"rm -rf /dir-to-delete", withDir, empty}, + } { + var buf bytes.Buffer + require.NoError(t, archive.WriteDiff(ctx, &buf, l.lower, l.upper)) + desc := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageLayer, + Digest: digest.FromBytes(buf.Bytes()), + Size: int64(buf.Len()), + } + blobs[desc.Digest] = buf.Bytes() + manifest.Layers = append(manifest.Layers, desc) + // The layer is uncompressed, so its diff ID is its blob digest. + config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, desc.Digest) + config.History = append(config.History, ocispec.History{CreatedBy: l.createdBy}) + } + + configBlob, err := json.Marshal(config) + require.NoError(t, err) + manifest.Config = ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageConfig, + Digest: digest.FromBytes(configBlob), + Size: int64(len(configBlob)), + } + blobs[manifest.Config.Digest] = configBlob + + manifest.MediaType = ocispec.MediaTypeImageManifest + manifestBlob, err := json.Marshal(manifest) + require.NoError(t, err) + target := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.FromBytes(manifestBlob), + Size: int64(len(manifestBlob)), + } + blobs[target.Digest] = manifestBlob + + return blobs, target +} + // TestIssue13030 is a regression test for parallel image unpacking. // The test validates that when multiple layers are unpacked in parallel, // that whiteout files are properly processed and do not cause files to @@ -1831,14 +1910,22 @@ func TestIssue13030(t *testing.T) { ctx, cancel := testContext(t) t.Cleanup(cancel) + // Pull the base image without unpacking it, so that all the layers of + // the derived image built below are unpacked in parallel. + base, err := client.Pull(ctx, images.Get(images.BusyBox), WithPlatformMatcher(platforms.Default())) + require.NoError(t, err) + + blobs, target := buildWhiteoutImage(ctx, t, client, base) + ref := testutil.ServeImage(t, "whiteout-test", "latest", target, blobs) + image, err := client.Pull(ctx, - images.Get(images.Whiteout), + ref, WithPlatformMatcher(platforms.Default()), WithPullUnpack, WithUnpackOpts([]UnpackOpt{WithUnpackLimiter(semaphore.NewWeighted(3))}), ) t.Cleanup(func() { - client.ImageService().Delete(ctx, images.Get(images.Whiteout)) + client.ImageService().Delete(ctx, ref) }) if err != nil { t.Fatal(err) diff --git a/integration/images/image_list.go b/integration/images/image_list.go index 632721ad16b5e..a2efed9361f30 100644 --- a/integration/images/image_list.go +++ b/integration/images/image_list.go @@ -39,7 +39,6 @@ type ImageList struct { VolumeOwnership string ArgsEscaped string Nginx string - Whiteout string } var ( @@ -60,7 +59,6 @@ func initImages(imageListFile string) { VolumeOwnership: "ghcr.io/containerd/volume-ownership:2.1", ArgsEscaped: "cplatpublic.azurecr.io/args-escaped-test-image-ns:1.0", Nginx: "ghcr.io/containerd/nginx:1.27.0", - Whiteout: "ghcr.io/containerd/whiteout-test:1.0", } if imageListFile != "" { @@ -102,8 +100,6 @@ const ( ArgsEscaped // Nginx image Nginx - // Whiteout image - Whiteout ) func initImageMap(imageList ImageList) map[int]string { @@ -117,7 +113,6 @@ func initImageMap(imageList ImageList) map[int]string { images[VolumeOwnership] = imageList.VolumeOwnership images[ArgsEscaped] = imageList.ArgsEscaped images[Nginx] = imageList.Nginx - images[Whiteout] = imageList.Whiteout return images } diff --git a/integration/images/whiteout-test/Dockerfile b/integration/images/whiteout-test/Dockerfile deleted file mode 100644 index d7b3483447a6c..0000000000000 --- a/integration/images/whiteout-test/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM alpine - -RUN touch /file-to-delete -RUN rm /file-to-delete - -RUN mkdir /dir-to-delete && touch /dir-to-delete/foo -RUN rm -rf /dir-to-delete diff --git a/integration/images/whiteout-test/Makefile b/integration/images/whiteout-test/Makefile deleted file mode 100644 index 02e7f3694a25f..0000000000000 --- a/integration/images/whiteout-test/Makefile +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright The containerd Authors. - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -all: build - -PROJ ?= ghcr.io/containerd -NAME ?= whiteout-test -VERSION ?= 1.0 -IMAGE ?= $(PROJ)/$(NAME):$(VERSION) - -DOCKER ?= docker -MIN_DOCKER_MAJOR ?= 29 -DOCKER_CLIENT_VERSION = $(shell $(DOCKER) version --format '{{.Client.Version}}' 2>/dev/null) -DOCKER_CLIENT_MAJOR = $(firstword $(subst ., ,$(DOCKER_CLIENT_VERSION))) - -# Use the same helper image as https://github.com/containerd/containerd/blob/main/test/init-buildx.sh#L78 -BINFMT_IMAGE ?= multiarch/qemu-user-static@sha256:c772ee1965aa0be9915ee1b018a0dd92ea361b4fa1bcab5bbc033517749b2af4 -PLATFORMS ?= linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/riscv64,linux/s390x - -# Optional/manual: register binfmt handlers for cross-arch emulation. -setup-binfmt: - $(DOCKER) run --privileged --rm $(BINFMT_IMAGE) --reset -p yes - -check-docker-version: - @if [ -z "$(DOCKER_CLIENT_VERSION)" ]; then \ - echo "error: failed to detect Docker client version"; \ - exit 1; \ - elif ! [ "$(DOCKER_CLIENT_MAJOR)" -eq "$(DOCKER_CLIENT_MAJOR)" ] 2>/dev/null; then \ - echo "error: failed to parse Docker major version from $(DOCKER_CLIENT_VERSION)"; \ - exit 1; \ - elif [ "$(DOCKER_CLIENT_MAJOR)" -lt "$(MIN_DOCKER_MAJOR)" ]; then \ - echo "error: this Makefile requires Docker >= $(MIN_DOCKER_MAJOR); found $(DOCKER_CLIENT_VERSION)"; \ - exit 1; \ - fi - -build: check-docker-version - $(DOCKER) buildx build --platform $(PLATFORMS) -t $(IMAGE) . - -push: build - $(DOCKER) push $(IMAGE) - -print-config: - @echo "DOCKER=$(DOCKER)" - @echo "MIN_DOCKER_MAJOR=$(MIN_DOCKER_MAJOR)" - @echo "DOCKER_CLIENT_VERSION=$(DOCKER_CLIENT_VERSION)" - @echo "IMAGE=$(IMAGE)" - @echo "PLATFORMS=$(PLATFORMS)" - -.PHONY: all setup-binfmt check-docker-version build push print-config diff --git a/internal/cri/server/container_checkpoint.go b/internal/cri/server/container_checkpoint.go index 3eebd89fd5a0e..1c433bc09e594 100644 --- a/internal/cri/server/container_checkpoint.go +++ b/internal/cri/server/container_checkpoint.go @@ -22,27 +22,11 @@ import ( "context" "time" - containerstore "github.com/containerd/containerd/v2/internal/cri/store/container" - "github.com/containerd/containerd/v2/internal/cri/store/sandbox" - "github.com/containerd/errdefs" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) -func (c *criService) checkIfCheckpointOCIImage(ctx context.Context, input string) (string, error) { - return "", nil -} - -func (c *criService) CRImportCheckpoint( - ctx context.Context, - meta *containerstore.Metadata, - sandbox *sandbox.Sandbox, - sandboxConfig *runtime.PodSandboxConfig, -) (ctrID string, retErr error) { - return "", errdefs.ErrNotImplemented -} - func (c *criService) CheckpointContainer(ctx context.Context, r *runtime.CheckpointContainerRequest) (res *runtime.CheckpointContainerResponse, err error) { // The next line is just needed to make the linter happy. containerCheckpointTimer.WithValues("no-runtime").UpdateSince(time.Now()) diff --git a/internal/cri/server/container_checkpoint_linux.go b/internal/cri/server/container_checkpoint_linux.go index bb81a57ccfa24..6d5927ee8a265 100644 --- a/internal/cri/server/container_checkpoint_linux.go +++ b/internal/cri/server/container_checkpoint_linux.go @@ -38,25 +38,13 @@ import ( "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/core/images" - "github.com/containerd/containerd/v2/core/mount" - "github.com/containerd/containerd/v2/internal/cri/annotations" - crilabels "github.com/containerd/containerd/v2/internal/cri/labels" - containerstore "github.com/containerd/containerd/v2/internal/cri/store/container" - imagestore "github.com/containerd/containerd/v2/internal/cri/store/image" - "github.com/containerd/containerd/v2/internal/cri/store/sandbox" "github.com/containerd/containerd/v2/pkg/archive" "github.com/containerd/containerd/v2/pkg/protobuf/proto" ptypes "github.com/containerd/containerd/v2/pkg/protobuf/types" "github.com/containerd/containerd/v2/plugins" - "github.com/containerd/continuity/fs" - "github.com/containerd/errdefs" "github.com/containerd/log" - "github.com/containerd/platforms" - "github.com/distribution/reference" - "github.com/opencontainers/image-spec/identity" v1 "github.com/opencontainers/image-spec/specs-go/v1" - spec "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" @@ -102,41 +90,6 @@ func copyNoFollow(src, dst string, perm os.FileMode) error { return err } -// checkpointArchiveEntryAllowed reports whether a tar entry from a checkpoint -// archive may be unpacked. Legitimate checkpoint archives contain only regular -// files and directories; other entry types (symlinks, hardlinks, device and fifo -// nodes) are not produced by the checkpoint code and are rejected as a hardening -// measure. -func checkpointArchiveEntryAllowed(hdr *tar.Header) bool { - switch hdr.Typeflag { - //nolint:staticcheck // TypeRegA is deprecated but we may still receive an external tar with TypeRegA - case tar.TypeReg, tar.TypeRegA, tar.TypeDir, tar.TypeXGlobalHeader: - return true - default: - return false - } -} - -// assertCheckpointDirSafe verifies that the populated restore directory contains -// only regular files and directories. -// -// The OCI-image restore path copies checkpoint content into the restore dir with -// fs.CopyDir, which (unlike the tar unpack filter) faithfully recreates any -// symlinks and special files present in the image. Restore-time consumers open -// paths under this directory, so non-regular entries are rejected before they run. -func assertCheckpointDirSafe(root string) error { - return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - // d.Type() reports the entry type without following symlinks. - if d.IsDir() || d.Type().IsRegular() { - return nil - } - return fmt.Errorf("refusing to restore checkpoint: %s is not a regular file or directory", path) - }) -} - func (c *criService) checkCriu() error { c.checkCriuOnce.Do(func() { c.checkCriuErr = c.doCheckCriu() @@ -189,412 +142,6 @@ func resolveCriuPath(customPath string) string { return "" } -// checkIfCheckpointOCIImage returns checks if the input refers to a checkpoint image. -// It returns the StorageImageID of the image the input resolves to, nil otherwise. -func (c *criService) checkIfCheckpointOCIImage(ctx context.Context, input string) (string, error) { - if input == "" { - return "", nil - } - if _, err := os.Stat(input); err == nil { - return "", nil - } - - image, err := c.LocalResolve(input) - if err != nil { - return "", fmt.Errorf("failed to resolve image %q: %w", input, err) - } - containerdImage, err := c.toContainerdImage(ctx, image) - if err != nil { - return "", fmt.Errorf("failed to get image from containerd %q: %w", input, err) - } - input = containerdImage.Name() - images, err := c.client.ImageService().Get(ctx, input) - if err != nil { - return "", fmt.Errorf("failed to get image from containerd %q: %w", input, err) - } - - rawIndex, err := content.ReadBlob(ctx, c.client.ContentStore(), images.Target) - if err != nil { - return "", fmt.Errorf("failed to read image blob from containerd %q: %w", input, err) - } - - var index v1.Index - if err = json.Unmarshal(rawIndex, &index); err != nil { - return "", fmt.Errorf("failed to unmarshall blob into OCI index: %w", err) - } - - if index.Annotations == nil { - return "", nil - } - - ann, ok := index.Annotations[crmetadata.CheckpointAnnotationName] - if !ok { - return "", nil - } - - log.G(ctx).Infof("Found checkpoint of container %v in %v", ann, input) - - return image.ID, nil -} - -func (c *criService) CRImportCheckpoint( - ctx context.Context, - meta *containerstore.Metadata, - sandbox *sandbox.Sandbox, - sandboxConfig *runtime.PodSandboxConfig, -) (ctrID string, retErr error) { - if err := c.checkCriu(); err != nil { - return "", fmt.Errorf("checkpoint restore is not enabled: %w", err) - } - - var mountPoint string - start := time.Now() - // Ensure that the image to restore the checkpoint from has been provided. - if meta.Config.Image == nil || meta.Config.Image.Image == "" { - return "", errors.New(`attribute "image" missing from container definition`) - } - - inputImage := meta.Config.Image.Image - createAnnotations := meta.Config.Annotations - createLabels := meta.Config.Labels - - restoreStorageImageID, err := c.checkIfCheckpointOCIImage(ctx, inputImage) - if err != nil { - return "", err - } - - mountPoint, err = os.MkdirTemp("", "checkpoint") - if err != nil { - return "", err - } - defer func() { - if err := os.RemoveAll(mountPoint); err != nil { - log.G(ctx).Errorf("Could not recursively remove %s: %q", mountPoint, err) - } - }() - var archiveFile *os.File - if restoreStorageImageID != "" { - log.G(ctx).Debugf("Restoring from oci image %s", inputImage) - platform, err := c.sandboxService.SandboxPlatform(ctx, sandbox.Sandboxer, sandbox.ID) - if err != nil { - return "", fmt.Errorf("failed to query sandbox platform: %w", err) - } - img, err := c.client.ImageService().Get(ctx, restoreStorageImageID) - if err != nil { - return "", err - } - - i := client.NewImageWithPlatform(c.client, img, platforms.Only(platform)) - diffIDs, err := i.RootFS(ctx) - if err != nil { - return "", err - } - chainID := identity.ChainID(diffIDs).String() - ociRuntime, err := c.config.GetSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler) - if err != nil { - return "", fmt.Errorf("failed to get sandbox runtime: %w", err) - } - s := c.client.SnapshotService(c.RuntimeSnapshotter(ctx, ociRuntime)) - - mounts, err := s.View(ctx, mountPoint, chainID) - if err != nil { - if errdefs.IsAlreadyExists(err) { - mounts, err = s.Mounts(ctx, mountPoint) - } - if err != nil { - return "", err - } - } - if err := mount.All(mounts, mountPoint); err != nil { - return "", err - } - } else { - - archiveFile, err = os.Open(inputImage) - if err != nil { - return "", fmt.Errorf("failed to open checkpoint archive %s for import: %w", inputImage, err) - } - defer func(f *os.File) { - if err := f.Close(); err != nil { - log.G(ctx).Errorf("Unable to close file %s: %q", f.Name(), err) - } - }(archiveFile) - - filter := archive.WithFilter(func(hdr *tar.Header) (bool, error) { - // Reject entry types the checkpoint code never produces (symlinks, - // hardlinks, device/fifo nodes) so they are not recreated on disk. - if !checkpointArchiveEntryAllowed(hdr) { - log.G(ctx).Warnf("Skipping unexpected checkpoint archive entry %q (type %d)", hdr.Name, hdr.Typeflag) - return false, nil - } - // The checkpoint archive is unpacked twice if using a tar file directly. - // The first time only the metadata files are relevant to prepare the - // restore operation. This filter function ignores the large parts of - // the checkpoint archive. This is usually the actual checkpoint - // coming from CRIU as well as the rootfs diff tar file. - excludePatterns := []string{ - "artifacts", - "ctr.log", - crmetadata.RootFsDiffTar, - crmetadata.NetworkStatusFile, - crmetadata.DeletedFilesFile, - crmetadata.CheckpointDirectory, - } - for _, pattern := range excludePatterns { - if strings.HasPrefix(hdr.Name, pattern) { - return false, nil - } - } - return true, nil - }) - - _, err = archive.Apply( - ctx, - mountPoint, - archiveFile, - []archive.ApplyOpt{filter}..., - ) - - if err != nil { - return "", fmt.Errorf("unpacking of checkpoint archive %s failed: %w", mountPoint, err) - } - log.G(ctx).Debugf("Unpacked checkpoint in %s", mountPoint) - } - // Load spec.dump from temporary directory - dumpSpec := new(spec.Spec) - if _, err := crmetadata.ReadJSONFile(dumpSpec, mountPoint, crmetadata.SpecDumpFile); err != nil { - return "", fmt.Errorf("failed to read %q: %w", crmetadata.SpecDumpFile, err) - } - - // Load config.dump from temporary directory - config := new(crmetadata.ContainerConfig) - if _, err := crmetadata.ReadJSONFile(config, mountPoint, crmetadata.ConfigDumpFile); err != nil { - return "", fmt.Errorf("failed to read %q: %w", crmetadata.ConfigDumpFile, err) - } - - // Load status.dump from temporary directory - containerStatus := new(runtime.ContainerStatus) - if _, err := crmetadata.ReadJSONFile(containerStatus, mountPoint, crmetadata.StatusDumpFile); err != nil { - return "", fmt.Errorf("failed to read %q: %w", crmetadata.StatusDumpFile, err) - } - - if meta.SandboxID == "" { - // restore into previous sandbox - meta.SandboxID = dumpSpec.Annotations[annotations.SandboxID] - ctrID = config.ID - } else { - ctrID = "" - } - - ctrMetadata := runtime.ContainerMetadata{} - - if meta.Config.Metadata != nil && meta.Config.Metadata.Name != "" { - ctrMetadata.Name = containerStatus.GetMetadata().GetName() - } - - originalAnnotations := containerStatus.GetAnnotations() - if originalAnnotations == nil { - originalAnnotations = make(map[string]string) - } - originalLabels := containerStatus.GetLabels() - - sandboxUID := sandboxConfig.GetMetadata().GetUid() - - if sandboxUID != "" { - if _, ok := originalLabels[crilabels.KubernetesPodUIDLabel]; ok { - originalLabels[crilabels.KubernetesPodUIDLabel] = sandboxUID - } - if _, ok := originalAnnotations[crilabels.KubernetesPodUIDLabel]; ok { - originalAnnotations[crilabels.KubernetesPodUIDLabel] = sandboxUID - } - } - - if createLabels != nil { - fixupLabels := []string{ - // Update the container name. It has already been update in metadata.Name. - // It also needs to be updated in the container labels. - crilabels.KubernetesContainerNameLabel, - // Update pod name in the labels. - crilabels.KubernetesPodNameLabel, - // Also update namespace. - crilabels.KubernetesPodNamespaceLabel, - } - - for _, annotation := range fixupLabels { - _, ok1 := createLabels[annotation] - _, ok2 := originalLabels[annotation] - - // If the value is not set in the original container or - // if it is not set in the new container, just skip - // the step of updating metadata. - if ok1 && ok2 { - originalLabels[annotation] = createLabels[annotation] - } - } - } - - originalAnnotations = filterAndMergeAnnotations( - ctx, - originalAnnotations, - createAnnotations, - ) - - var containerdImage client.Image - - containerdImage, err = c.client.GetImage(ctx, config.RootfsImageRef) - if err != nil { - if !errdefs.IsNotFound(err) { - return "", fmt.Errorf("failed to get checkpoint base image %s: %w", config.RootfsImageRef, err) - } - // Pulling the image the checkpoint is based on. This is a bit different - // than automatic image pulling. The checkpoint image is not automatically - // pulled, but the image the checkpoint is based on. - // During checkpointing the base image of the checkpoint is stored in the - // checkpoint archive as NAME@DIGEST. The checkpoint archive also contains - // the tag with which it was initially pulled. - // First step is to pull NAME@DIGEST - containerdImage, err = c.client.Pull(ctx, config.RootfsImageRef) - if err != nil { - return "", fmt.Errorf("failed to pull checkpoint base image %s: %w", config.RootfsImageRef, err) - } - } - - if _, err := reference.ParseAnyReference(config.RootfsImageName); err != nil { - return "", fmt.Errorf("error parsing reference: %q is not a valid repository/tag %v", config.RootfsImageName, err) - } - - var image imagestore.Image - for i := 1; i < 500; i++ { - // This is probably wrong. Not sure how to wait for an image to appear in - // the image (or content) store. - log.G(ctx).Debugf("Trying to resolve %s:%d", containerdImage.Name(), i) - image, err = c.LocalResolve(containerdImage.Name()) - if err == nil { - break - } - time.Sleep(time.Microsecond * time.Duration(i)) - } - if err != nil { - return "", fmt.Errorf("failed to resolve image %q during checkpoint import: %w", config.RootfsImageName, err) - } - imageConfig := image.ImageSpec.Config - env := append([]string{}, imageConfig.Env...) - for _, e := range meta.Config.GetEnvs() { - env = append(env, e.GetKey()+"="+string(e.GetValue())) - } - imageConfig.Env = env - - originalAnnotations["restored"] = "true" - originalAnnotations["checkpointedAt"] = config.CheckpointedAt.Format(time.RFC3339Nano) - originalAnnotations["checkpointImage"] = meta.Config.Image.GetUserSpecifiedImage() - - meta.Config.Annotations = originalAnnotations - - // Remove the checkpoint image name and show the base image name in the metadata. - // The checkpoint image name is still available in the annotations. - meta.Config.Image.Image = containerStatus.Image.GetImage() - meta.Config.Image.UserSpecifiedImage = containerStatus.Image.GetUserSpecifiedImage() - - cstatus, err := c.sandboxService.SandboxStatus(ctx, sandbox.Sandboxer, sandbox.ID, false) - if err != nil { - return "", fmt.Errorf("failed to get controller status: %w", err) - } - - containerRootDir, err := c.createContainer( - &createContainerRequest{ - ctx: ctx, - containerID: meta.ID, - sandbox: sandbox, - sandboxID: meta.SandboxID, - imageID: image.ID, - containerConfig: meta.Config, - imageConfig: &imageConfig, - podSandboxConfig: sandboxConfig, - sandboxRuntimeHandler: sandbox.Metadata.RuntimeHandler, - sandboxPid: cstatus.Pid, - NetNSPath: sandbox.NetNSPath, - containerName: containerName, - containerdImage: &containerdImage, - meta: meta, - restore: true, - start: start, - }, - ) - if err != nil { - return "", err - } - - // Confine all checkpoint content to a dedicated subdirectory of the container - // state dir instead of unpacking it directly into the state dir, so it cannot - // collide with containerd's own files there. Create it fresh; RemoveAll unlinks - // any pre-existing entry without following it. - restoreDir := filepath.Join(containerRootDir, checkpointRestoreDir) - if err := os.RemoveAll(restoreDir); err != nil { - return "", err - } - if err := os.Mkdir(restoreDir, 0o700); err != nil { - return "", err - } - - if restoreStorageImageID != "" { - if err := fs.CopyDir(restoreDir, mountPoint); err != nil { - return "", err - } - if err := mount.UnmountAll(mountPoint, 0); err != nil { - return "", err - } - // fs.CopyDir recreates any symlinks/special files from the image; reject - // them here so restore-time consumers only ever open regular files. - if err := assertCheckpointDirSafe(restoreDir); err != nil { - return "", err - } - } else { - // unpack the checkpoint archive - filter := archive.WithFilter(func(hdr *tar.Header) (bool, error) { - // Reject entry types the checkpoint code never produces (symlinks, - // hardlinks, device/fifo nodes) so they are not recreated on disk. - if !checkpointArchiveEntryAllowed(hdr) { - log.G(ctx).Warnf("Skipping unexpected checkpoint archive entry %q (type %d)", hdr.Name, hdr.Typeflag) - return false, nil - } - excludePatterns := []string{ - crmetadata.ConfigDumpFile, - crmetadata.SpecDumpFile, - crmetadata.StatusDumpFile, - } - - for _, pattern := range excludePatterns { - if strings.HasPrefix(hdr.Name, pattern) { - return false, nil - } - } - - return true, nil - }) - - // Start from the beginning of the checkpoint archive - archiveFile.Seek(0, 0) - _, err = archive.Apply(ctx, restoreDir, archiveFile, []archive.ApplyOpt{filter}...) - - if err != nil { - return "", fmt.Errorf("unpacking of checkpoint archive %s failed: %w", restoreDir, err) - } - } - log.G(ctx).Debugf("Unpacked checkpoint in %s", restoreDir) - - // Restore container log file (if it exists). - // - // container.log was unpacked from a checkpoint archive/OCI image, so it is - // copied without following a final-component symlink. - containerLog := filepath.Join(restoreDir, "container.log") - if err := copyNoFollow(containerLog, meta.LogPath, 0600); err != nil { - if !errors.Is(err, os.ErrNotExist) { - return "", fmt.Errorf("restoring container log file %s failed: %w", containerLog, err) - } - } - return meta.ID, nil -} - func (c *criService) CheckpointContainer(ctx context.Context, r *runtime.CheckpointContainerRequest) (*runtime.CheckpointContainerResponse, error) { start := time.Now() if err := c.checkCriu(); err != nil { @@ -879,37 +426,3 @@ func writeSpecDumpFile(ctx context.Context, store content.Store, desc v1.Descrip return nil } - -func filterAndMergeAnnotations( - ctx context.Context, - checkpointAnnotations map[string]string, - createAnnotations map[string]string, -) map[string]string { - result := make(map[string]string) - - for k, v := range checkpointAnnotations { - if strings.HasPrefix(k, "cdi.k8s.io/") || k == "cdi.k8s.io" { - log.G(ctx).Warnf("Denying annotation %q in checkpoint restore", k) - continue - } - result[k] = v - } - - // The hash also needs to be update or Kubernetes thinks the container needs to be restarted - _, ok1 := createAnnotations["io.kubernetes.container.hash"] - _, ok2 := result["io.kubernetes.container.hash"] - - if ok1 && ok2 { - result["io.kubernetes.container.hash"] = createAnnotations["io.kubernetes.container.hash"] - } - - // The restart count also needs to be correctly updated - _, ok1 = createAnnotations["io.kubernetes.container.restartCount"] - _, ok2 = result["io.kubernetes.container.restartCount"] - - if ok1 && ok2 { - result["io.kubernetes.container.restartCount"] = createAnnotations["io.kubernetes.container.restartCount"] - } - - return result -} diff --git a/internal/cri/server/container_checkpoint_linux_test.go b/internal/cri/server/container_checkpoint_linux_test.go index defcc3f440ad3..1eb7448204499 100644 --- a/internal/cri/server/container_checkpoint_linux_test.go +++ b/internal/cri/server/container_checkpoint_linux_test.go @@ -19,17 +19,13 @@ package server import ( - "archive/tar" "context" "errors" "os" "path/filepath" "strings" - "sync" "testing" - "github.com/containerd/log" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -107,137 +103,6 @@ func TestCopyNoFollowRejectsDirectory(t *testing.T) { assert.NoFileExists(t, dst) } -func TestAssertCheckpointDirSafe(t *testing.T) { - t.Run("regular files and dirs allowed", func(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(root, "checkpoint"), 0o700)) - require.NoError(t, os.WriteFile(filepath.Join(root, "checkpoint", "img"), []byte("x"), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(root, "rootfs-diff.tar"), []byte("x"), 0o600)) - assert.NoError(t, assertCheckpointDirSafe(root)) - }) - - t.Run("symlink rejected", func(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.Symlink("/some/outside/path", filepath.Join(root, "rootfs-diff.tar"))) - assert.Error(t, assertCheckpointDirSafe(root)) - }) - - t.Run("symlink nested in subdir rejected", func(t *testing.T) { - root := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(root, "checkpoint"), 0o700)) - require.NoError(t, os.Symlink("/some/outside/path", filepath.Join(root, "checkpoint", "pages-1.img"))) - assert.Error(t, assertCheckpointDirSafe(root)) - }) - - t.Run("fifo rejected", func(t *testing.T) { - root := t.TempDir() - require.NoError(t, unix.Mkfifo(filepath.Join(root, "fifo"), 0o600)) - assert.Error(t, assertCheckpointDirSafe(root)) - }) -} - -func TestCheckpointArchiveEntryAllowed(t *testing.T) { - for _, tc := range []struct { - name string - typ byte - allowed bool - }{ - {"regular", tar.TypeReg, true}, - //nolint:staticcheck // TypeRegA is deprecated but external tars may still use it - {"regular-A", tar.TypeRegA, true}, - {"directory", tar.TypeDir, true}, - {"global-header", tar.TypeXGlobalHeader, true}, - {"symlink", tar.TypeSymlink, false}, - {"hardlink", tar.TypeLink, false}, - {"char-device", tar.TypeChar, false}, - {"block-device", tar.TypeBlock, false}, - {"fifo", tar.TypeFifo, false}, - } { - t.Run(tc.name, func(t *testing.T) { - got := checkpointArchiveEntryAllowed(&tar.Header{Typeflag: tc.typ, Name: tc.name}) - assert.Equal(t, tc.allowed, got) - }) - } -} - -type testLogHook struct { - mu sync.Mutex - entries []string -} - -func (h *testLogHook) Levels() []logrus.Level { - return []logrus.Level{logrus.WarnLevel} -} - -func (h *testLogHook) Fire(entry *logrus.Entry) error { - h.mu.Lock() - defer h.mu.Unlock() - h.entries = append(h.entries, entry.Message) - return nil -} - -func TestFilterAndMergeAnnotations(t *testing.T) { - for desc, tc := range map[string]struct { - checkpointAnnotations map[string]string - createAnnotations map[string]string - expectedAnnotations map[string]string - expectedWarnings []string - }{ - "cdi denied prefix boundaries": { - checkpointAnnotations: map[string]string{ - "cdi.k8s.io/device": "gpu", - "cdi.k8s.io/": "true", - "cdi.k8s.io": "true", - "safe.org/cdi.k8s.io": "ignored", - "other": "val", - }, - expectedAnnotations: map[string]string{ - "safe.org/cdi.k8s.io": "ignored", - "other": "val", - }, - expectedWarnings: []string{ - `Denying annotation "cdi.k8s.io/device" in checkpoint restore`, - `Denying annotation "cdi.k8s.io/" in checkpoint restore`, - `Denying annotation "cdi.k8s.io" in checkpoint restore`, - }, - }, - - "createAnnotations update kubernetes metadata if present in both": { - checkpointAnnotations: map[string]string{ - "io.kubernetes.container.hash": "old-hash", - "io.kubernetes.container.restartCount": "1", - "safe.annotation": "2", - }, - createAnnotations: map[string]string{ - "io.kubernetes.container.hash": "new-hash", - "io.kubernetes.container.restartCount": "2", - }, - expectedAnnotations: map[string]string{ - "io.kubernetes.container.hash": "new-hash", - "io.kubernetes.container.restartCount": "2", - "safe.annotation": "2", - }, - }, - } { - t.Run(desc, func(t *testing.T) { - logger := logrus.New() - logger.SetLevel(logrus.WarnLevel) - hook := &testLogHook{} - logger.AddHook(hook) - ctx := log.WithLogger(context.Background(), logrus.NewEntry(logger)) - - res := filterAndMergeAnnotations( - ctx, - tc.checkpointAnnotations, - tc.createAnnotations, - ) - - assert.Equal(t, tc.expectedAnnotations, res) - assert.ElementsMatch(t, tc.expectedWarnings, hook.entries) - }) - } -} - func TestResolveCriuPath(t *testing.T) { tempDir, err := os.MkdirTemp("", "criu-test") if err != nil { @@ -357,15 +222,3 @@ func TestCheckpointContainerDisabled(t *testing.T) { t.Errorf("expected error containing 'criu support is disabled by configuration', got: %v", err) } } - -func TestCRImportCheckpointDisabled(t *testing.T) { - c := newTestCRIService() - c.config.EnableCRIU = func() *bool { v := false; return &v }() - _, err := c.CRImportCheckpoint(context.Background(), nil, nil, nil) - if err == nil { - t.Fatal("expected error, got nil") - } - if !strings.Contains(err.Error(), "criu support is disabled by configuration") { - t.Errorf("expected error containing 'criu support is disabled by configuration', got: %v", err) - } -} diff --git a/internal/cri/server/container_checkpoint_warning_test.go b/internal/cri/server/container_checkpoint_warning_test.go deleted file mode 100644 index 88e9665c2a843..0000000000000 --- a/internal/cri/server/container_checkpoint_warning_test.go +++ /dev/null @@ -1,91 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package server - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - - "github.com/containerd/containerd/v2/core/sandbox" - sandboxstore "github.com/containerd/containerd/v2/internal/cri/store/sandbox" - "github.com/containerd/containerd/v2/pkg/deprecation" - "github.com/containerd/containerd/v2/plugins/services/warning" -) - -type mockWarningService struct { - emitted []deprecation.Warning -} - -func (m *mockWarningService) Emit(ctx context.Context, w deprecation.Warning) { - m.emitted = append(m.emitted, w) -} - -func (m *mockWarningService) Warnings() []warning.Warning { - return nil -} - -type testSandboxService struct { - fakeSandboxService -} - -func (t *testSandboxService) SandboxStatus(ctx context.Context, sandboxer string, sandboxID string, verbose bool) (sandbox.ControllerStatus, error) { - return sandbox.ControllerStatus{ - SandboxID: sandboxID, - Pid: 1234, - State: "READY", - }, nil -} - -func TestCreateContainerCheckpointWarning(t *testing.T) { - c := newTestCRIService() - mockWarn := &mockWarningService{} - c.warningService = mockWarn - c.sandboxService = &testSandboxService{} - - sb := sandboxstore.NewSandbox( - sandboxstore.Metadata{ - ID: "test-sandbox", - Name: "test-sandbox", - Config: &runtime.PodSandboxConfig{ - Metadata: &runtime.PodSandboxMetadata{Name: "test-sandbox", Namespace: "default"}, - }, - }, - sandboxstore.Status{ - State: sandboxstore.StateReady, - }, - ) - require.NoError(t, c.sandboxStore.Add(sb)) - - // In newTestCRIService(), c.os is a FakeOS where Stat returns (nil, nil) (no error), - // causing checkpointImage to evaluate as true when checked in CreateContainer. - _, _ = c.CreateContainer(context.Background(), &runtime.CreateContainerRequest{ - PodSandboxId: "test-sandbox", - Config: &runtime.ContainerConfig{ - Metadata: &runtime.ContainerMetadata{Name: "test-container"}, - Image: &runtime.ImageSpec{Image: "/path/to/checkpoint.tar"}, - }, - SandboxConfig: &runtime.PodSandboxConfig{ - Metadata: &runtime.PodSandboxMetadata{Name: "test-sandbox", Namespace: "default"}, - }, - }) - - assert.Contains(t, mockWarn.emitted, deprecation.CRICreateContainerCheckpointRestore, "expected CRICreateContainerCheckpointRestore deprecation warning to be emitted") -} diff --git a/internal/cri/server/container_create.go b/internal/cri/server/container_create.go index 376628afe9f10..3e5268656502f 100644 --- a/internal/cri/server/container_create.go +++ b/internal/cri/server/container_create.go @@ -47,7 +47,6 @@ import ( "github.com/containerd/containerd/v2/internal/cri/util" "github.com/containerd/containerd/v2/internal/registrar" "github.com/containerd/containerd/v2/pkg/blockio" - "github.com/containerd/containerd/v2/pkg/deprecation" "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/containerd/v2/pkg/tracing" ) @@ -129,63 +128,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta Config: config, } - // Check if image is a file. If it is a file it might be a checkpoint archive. - checkpointImage, err := func() (bool, error) { - if _, err := c.os.Stat(config.GetImage().GetImage()); err == nil { - log.G(ctx).Infof( - "%q is a file. Assuming it is a checkpoint archive", - config.GetImage().GetImage(), - ) - return true, nil - } - // Check if this is an OCI checkpoint image - imageID, err := c.checkIfCheckpointOCIImage(ctx, config.GetImage().GetImage()) - if err != nil { - return false, fmt.Errorf("failed to check if this is a checkpoint image: %w", err) - } - - return imageID != "", nil - }() - if err != nil { - return nil, err - } - - if checkpointImage { - // This might be a checkpoint image. Let's pass - // it to the checkpoint code. - - if c.warningService != nil { - c.warningService.Emit(ctx, deprecation.CRICreateContainerCheckpointRestore) - if msg, ok := deprecation.Message(deprecation.CRICreateContainerCheckpointRestore); ok { - log.G(ctx).WithFields(log.Fields{ - "podsandboxid": sandboxID, - "containerid": id, - "containername": name, - }).Warn(msg) - } - } - - if sandboxConfig.GetMetadata() == nil { - return nil, fmt.Errorf("sandboxConfig must not be empty") - } - - ctrID, err := c.CRImportCheckpoint( - ctx, - &meta, - &sandbox, - sandboxConfig, - ) - if err != nil { - log.G(ctx).Errorf("failed to prepare %s for restore %q", ctrID, err) - return nil, err - } - log.G(ctx).Infof("Prepared %s for restore", ctrID) - - return &runtime.CreateContainerResponse{ - ContainerId: id, - }, nil - } - // Prepare container image snapshot. For container, the image should have // been pulled before creating the container, so do not ensure the image. image, err := c.LocalResolve(config.GetImage().GetImage()) @@ -242,7 +184,6 @@ type createContainerRequest struct { containerName string containerdImage *containerd.Image meta *containerstore.Metadata - restore bool start time.Time } @@ -472,7 +413,7 @@ func (c *criService) createContainer(r *createContainerRequest) (_ string, retEr } }() - status := containerstore.Status{CreatedAt: time.Now().UnixNano(), Restore: r.restore} + status := containerstore.Status{CreatedAt: time.Now().UnixNano()} status = copyResourcesToStatus(spec, status) container, err := containerstore.NewContainer(*r.meta, containerstore.WithStatus(status, containerRootDir), diff --git a/internal/cri/server/container_start.go b/internal/cri/server/container_start.go index 0c7653cade4de..0d3567d46de3a 100644 --- a/internal/cri/server/container_start.go +++ b/internal/cri/server/container_start.go @@ -21,8 +21,6 @@ import ( "errors" "fmt" "io" - "os" - "path/filepath" "time" "github.com/containerd/containerd/v2/pkg/tracing" @@ -30,7 +28,6 @@ import ( "github.com/containerd/log" runtime "k8s.io/cri-api/pkg/apis/runtime/v1" - crmetadata "github.com/checkpoint-restore/checkpointctl/lib" containerd "github.com/containerd/containerd/v2/client" cio "github.com/containerd/containerd/v2/internal/cri/io" containerstore "github.com/containerd/containerd/v2/internal/cri/store/container" @@ -40,12 +37,6 @@ import ( cioutil "github.com/containerd/containerd/v2/pkg/ioutil" ) -// checkpointRestoreDir is the subdirectory under a container's persistent state -// directory into which checkpoint content (CRIU images, container.log, -// rootfs-diff.tar, ...) is unpacked during restore. Confining it here keeps -// checkpoint content from colliding with containerd's own files in the state dir. -const checkpointRestoreDir = "ctrd-restore" - // StartContainer starts the container. func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContainerRequest) (retRes *runtime.StartContainerResponse, retErr error) { span := tracing.SpanFromContext(ctx) @@ -110,65 +101,6 @@ func (c *criService) StartContainer(ctx context.Context, r *runtime.StartContain return cntr.IO, nil } - if cntr.Status.Get().Restore { - // If during start the container is detected as a checkpoint the container - // will be marked with Restore() == true. In this case not the normal - // start code is needed but this code which does a restore. - pid, err := container.Restore( - ctx, - ioCreation, - filepath.Join(c.getContainerRootDir(r.GetContainerId()), checkpointRestoreDir, crmetadata.CheckpointDirectory), - ) - - if err != nil { - return nil, fmt.Errorf("failed to restore containerd task: %w", err) - } - // Update container start timestamp. - if err := cntr.Status.UpdateSync(func(status containerstore.Status) (containerstore.Status, error) { - if pid < 0 { - return status, fmt.Errorf("restore returned a PID < 0 (%d); that should not happen", pid) - } - status.Pid = uint32(pid) - status.StartedAt = time.Now().UnixNano() - return status, nil - }); err != nil { - return nil, fmt.Errorf("failed to update container %q state: %w", id, err) - } - - c.generateAndSendContainerEvent(ctx, id, sandboxID, runtime.ContainerEventType_CONTAINER_STARTED_EVENT) - - task, err := cntr.Container.Task(ctx, nil) - if err != nil { - return nil, fmt.Errorf("failed to get task for container %q: %w", id, err) - } - // wait is a long running background request, no timeout needed. - exitCh, err := task.Wait(ctrdutil.NamespacedContext()) - if err != nil { - return nil, fmt.Errorf("failed to wait for containerd task: %w", err) - } - - defer func() { - if retErr != nil { - deferCtx, deferCancel := ctrdutil.DeferContext() - defer deferCancel() - err = c.nri.StopContainer(deferCtx, &sandbox, &cntr) - if err != nil { - log.G(ctx).WithError(err).Errorf("NRI stop failed for failed container %q", id) - } - } - }() - // It handles the TaskExit event and update container state after this. - c.startContainerExitMonitor(context.Background(), id, task.Pid(), exitCh) - - // cleanup checkpoint artifacts after restore. - restoreDir := filepath.Join(c.getContainerRootDir(r.GetContainerId()), checkpointRestoreDir) - if err := os.RemoveAll(restoreDir); err != nil { - log.G(ctx).Warnf("Non-fatal: removal of checkpoint restore dir (%s) failed: %v", restoreDir, err) - } - - log.G(ctx).Infof("Restored container %s successfully", r.GetContainerId()) - return &runtime.StartContainerResponse{}, nil - } // Recheck target container validity in Linux namespace options. if linux := config.GetLinux(); linux != nil { nsOpts := linux.GetSecurityContext().GetNamespaceOptions() diff --git a/internal/cri/store/container/status.go b/internal/cri/store/container/status.go index e1b618756c56d..e952b2798fba2 100644 --- a/internal/cri/store/container/status.go +++ b/internal/cri/store/container/status.go @@ -98,9 +98,6 @@ type Status struct { Unknown bool `json:"-"` // Resources has container runtime resource constraints Resources *runtime.ContainerResources - // Restore marks this container as a container to be restored from a - // checkpoint and not started. - Restore bool } // State returns current state of the container based on the container status. diff --git a/pkg/deprecation/deprecation.go b/pkg/deprecation/deprecation.go index ef9999e61e319..45a07f4198463 100644 --- a/pkg/deprecation/deprecation.go +++ b/pkg/deprecation/deprecation.go @@ -43,8 +43,6 @@ const ( RuncOptionsTaskAPIAddress Warning = Prefix + "runc-options-task-api-address" // RuncOptionsTaskAPIVersion is a warning for the use of `task_api_version` in runc options RuncOptionsTaskAPIVersion Warning = Prefix + "runc-options-task-api-version" - // CRICreateContainerCheckpointRestore is a warning for restoring checkpoint data from an image or archive during CRI CreateContainer - CRICreateContainerCheckpointRestore Warning = Prefix + "cri-create-container-checkpoint-restore" ) const ( @@ -69,9 +67,8 @@ var messages = map[Warning]string{ CgroupV1: "The support for cgroup v1 is deprecated since containerd v2.2 and will be removed by no later than May 2029. Upgrade the host to use cgroup v2.", CRIEnableCDI: "The `enable_cdi` property of `[plugins.\"io.containerd.cri.v1.runtime\"]` is deprecated, will be removed in containerd v2.3, and CDI support will always be enabled.", - RuncOptionsTaskAPIAddress: "The `task_api_address` field in runc options is deprecated since containerd v2.3. Set `task_api_address` on CreateTaskRequest instead.", - RuncOptionsTaskAPIVersion: "The `task_api_version` field in runc options is deprecated since containerd v2.3. Set `task_api_version` on CreateTaskRequest instead.", - CRICreateContainerCheckpointRestore: "Restoring checkpoint data from an image or archive during CRI CreateContainer is deprecated and will be removed in containerd v2.4.", + RuncOptionsTaskAPIAddress: "The `task_api_address` field in runc options is deprecated since containerd v2.3. Set `task_api_address` on CreateTaskRequest instead.", + RuncOptionsTaskAPIVersion: "The `task_api_version` field in runc options is deprecated since containerd v2.3. Set `task_api_version` on CreateTaskRequest instead.", } // Valid checks whether a given Warning is valid diff --git a/pkg/testutil/registry.go b/pkg/testutil/registry.go new file mode 100644 index 0000000000000..d86ed0899808c --- /dev/null +++ b/pkg/testutil/registry.go @@ -0,0 +1,92 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testutil + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ServeImage serves an image from a read-only HTTP registry listening on +// localhost, and returns a reference to pull the image from. +// +// blobs must be keyed by digest and contain the manifest blob referenced by +// target, along with the blobs it references, except for those already +// present in the puller's content store. The registry is shut down when the +// test completes. +func ServeImage(t *testing.T, name, tag string, target ocispec.Descriptor, blobs map[digest.Digest][]byte) string { + manifestPath := "/v2/" + name + "/manifests/" + blobPath := "/v2/" + name + "/blobs/" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet && r.Method != http.MethodHead { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + var b []byte + mediaType := "application/octet-stream" + switch { + case strings.HasPrefix(r.URL.Path, manifestPath): + // A manifest may be requested by the tag, by the target + // digest, or, for a multi-platform image, by the digest of + // a platform manifest. + dgst := target.Digest + if ref := strings.TrimPrefix(r.URL.Path, manifestPath); ref != tag { + var err error + if dgst, err = digest.Parse(ref); err != nil { + w.WriteHeader(http.StatusNotFound) + return + } + } + var ok bool + if b, ok = blobs[dgst]; !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if dgst == target.Digest { + mediaType = target.MediaType + } + w.Header().Set("Docker-Content-Digest", dgst.String()) + case strings.HasPrefix(r.URL.Path, blobPath): + dgst, err := digest.Parse(strings.TrimPrefix(r.URL.Path, blobPath)) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + var ok bool + if b, ok = blobs[dgst]; !ok { + w.WriteHeader(http.StatusNotFound) + return + } + default: + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", mediaType) + w.Header().Set("Content-Length", strconv.Itoa(len(b))) + if r.Method == http.MethodGet { + w.Write(b) + } + })) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://") + "/" + name + ":" + tag +}