From 415856f1a7344e0dcdc8b609d06dac6025f7e326 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Mon, 22 Jul 2024 17:32:50 +0800 Subject: [PATCH 01/11] tests: remove old tests Currently, all integration tests fail F40 because of failing to install package and enable services on the cloud image properly. TMT (Test Management Tool) supports running tests on virtual machines and provides an user-friendly interface (YAML) to define tasks like installing packages, running commands on the virtual machines. By using TMT, we no longer need to spend time and efforts on maintaining the current test framework and can also enjoy rich handy features like managing tests provided by TMT. Signed-off-by: Coiby Xu --- tests/Makefile | 88 ------ tests/README | 65 ----- tests/scripts/build-image.sh | 57 ---- tests/scripts/build-scripts/base-image.sh | 9 - .../scripts/build-scripts/test-base-image.sh | 21 -- tests/scripts/build-scripts/test-image.sh | 21 -- tests/scripts/copy-from-image.sh | 25 -- tests/scripts/image-init-lib.sh | 275 ------------------ tests/scripts/kexec-kdump-test/init.sh | 122 -------- .../kexec-kdump-test/kexec-kdump-test.service | 9 - tests/scripts/kexec-kdump-test/test.sh | 15 - tests/scripts/run-qemu | 23 -- tests/scripts/run-test.sh | 176 ----------- tests/scripts/spawn-image-shell.sh | 16 - tests/scripts/test-lib.sh | 176 ----------- .../scripts/testcases/local-kdump/0-local.sh | 32 -- .../lvm2-thinp-kdump/0-local-lvm2-thinp.sh | 59 ---- .../testcases/lvm2-thinp-kdump/lvm.conf | 5 - .../testcases/nfs-early-kdump/0-server.sh | 1 - .../testcases/nfs-early-kdump/1-client.sh | 46 --- tests/scripts/testcases/nfs-kdump/0-server.sh | 42 --- tests/scripts/testcases/nfs-kdump/1-client.sh | 30 -- tests/scripts/testcases/ssh-kdump/0-server.sh | 38 --- tests/scripts/testcases/ssh-kdump/1-client.sh | 42 --- 24 files changed, 1393 deletions(-) delete mode 100644 tests/Makefile delete mode 100644 tests/README delete mode 100755 tests/scripts/build-image.sh delete mode 100755 tests/scripts/build-scripts/base-image.sh delete mode 100755 tests/scripts/build-scripts/test-base-image.sh delete mode 100755 tests/scripts/build-scripts/test-image.sh delete mode 100755 tests/scripts/copy-from-image.sh delete mode 100644 tests/scripts/image-init-lib.sh delete mode 100755 tests/scripts/kexec-kdump-test/init.sh delete mode 100644 tests/scripts/kexec-kdump-test/kexec-kdump-test.service delete mode 100755 tests/scripts/kexec-kdump-test/test.sh delete mode 100755 tests/scripts/run-qemu delete mode 100755 tests/scripts/run-test.sh delete mode 100755 tests/scripts/spawn-image-shell.sh delete mode 100644 tests/scripts/test-lib.sh delete mode 100755 tests/scripts/testcases/local-kdump/0-local.sh delete mode 100755 tests/scripts/testcases/lvm2-thinp-kdump/0-local-lvm2-thinp.sh delete mode 100644 tests/scripts/testcases/lvm2-thinp-kdump/lvm.conf delete mode 120000 tests/scripts/testcases/nfs-early-kdump/0-server.sh delete mode 100755 tests/scripts/testcases/nfs-early-kdump/1-client.sh delete mode 100755 tests/scripts/testcases/nfs-kdump/0-server.sh delete mode 100755 tests/scripts/testcases/nfs-kdump/1-client.sh delete mode 100755 tests/scripts/testcases/ssh-kdump/0-server.sh delete mode 100755 tests/scripts/testcases/ssh-kdump/1-client.sh diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index 45688caf..00000000 --- a/tests/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -TEST_CASE ?= -BASE_IMAGE ?= - -TEST_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) -BUILD_ROOT := $(TEST_ROOT)/build -REPO = $(shell realpath $(TEST_ROOT)/../) -ARCH ?= $(shell arch) -SPEC = kexec-tools.spec - -DIST ?= fedora -DIST_ABR ?= f -DIST_ABRL ?= fc -DIST_UNSET ?= rhel -RELEASE ?= 33 - -DEFAULT_BASE_IMAGE_VER ?= 1.2 -DEFAULT_BASE_IMAGE ?= Fedora-Cloud-Base-$(RELEASE)-$(DEFAULT_BASE_IMAGE_VER).$(ARCH).raw.xz -DEFAULT_BASE_IMAGE_URL ?= https://dl.fedoraproject.org/pub/fedora/linux/releases/$(RELEASE)/Cloud/$(ARCH)/images/$(DEFAULT_BASE_IMAGE) - -BUILD_ROOT = $(TEST_ROOT)/build -RPMDEFINE = --define '_sourcedir $(REPO)'\ - --define '_specdir $(REPO)'\ - --define '_builddir $(BUILD_ROOT)'\ - --define '_srcrpmdir $(BUILD_ROOT)'\ - --define '_rpmdir $(BUILD_ROOT)'\ - --define 'dist %{?distprefix}.$(DIST_ABRL)$(RELEASE)'\ - --define '$(DIST) $(RELEASE)'\ - --eval '%undefine $(DIST_UNSET)'\ - --define '$(DIST_ABRL)$(RELEASE) 1'\ - -KEXEC_TOOLS_SRC = $(filter-out $(REPO)/tests,$(wildcard $(REPO)/*)) -KEXEC_TOOLS_TEST_SRC = $(wildcard $(REPO)/tests/scripts/**/*) -KEXEC_TOOLS_NVR = $(shell rpm $(RPMDEFINE) -q --specfile $(REPO)/$(SPEC) 2>/dev/null | grep -m 1 . | sed -e 's#.src#.$(ARCH)#') -KEXEC_TOOLS_RPM = $(BUILD_ROOT)/$(ARCH)/$(KEXEC_TOOLS_NVR).rpm - -all: $(TEST_ROOT)/output/test-base-image - -# Use either: -# fedpkg --release $(DIST_ABR)$(RELEASE) --path ../../ local -# or -# rpmbuild $(RPMDEFINE) -ba $(REPO)/$(SPEC) -# to rebuild the rpm, currently use rpmbuild to have better control over the rpm building process -# -$(KEXEC_TOOLS_RPM): $(KEXEC_TOOLS_SRC) - sh -c "cd .. && fedpkg --release f$(RELEASE) sources" - @echo Rebuilding RPM due to modification of sources: $? - rpmbuild $(RPMDEFINE) -ba $(REPO)/$(SPEC) - -$(BUILD_ROOT)/base-image: - mkdir -p $(BUILD_ROOT) -ifeq ($(strip $(BASE_IMAGE)),) - wget $(DEFAULT_BASE_IMAGE_URL) -O $(BUILD_ROOT)/$(DEFAULT_BASE_IMAGE) - $(TEST_ROOT)/scripts/build-image.sh \ - $(BUILD_ROOT)/$(DEFAULT_BASE_IMAGE)\ - $(BUILD_ROOT)/base-image -else - $(TEST_ROOT)/scripts/build-image.sh \ - $(BASE_IMAGE)\ - $(BUILD_ROOT)/base-image -endif - -$(BUILD_ROOT)/inst-base-image: $(BUILD_ROOT)/base-image - @echo "Building installation base image" - echo $(KEXEC_TOOLS_NVR) - $(TEST_ROOT)/scripts/build-image.sh \ - $(BUILD_ROOT)/base-image \ - $(BUILD_ROOT)/inst-base-image \ - $(TEST_ROOT)/scripts/build-scripts/base-image.sh - -$(TEST_ROOT)/output/test-base-image: $(BUILD_ROOT)/inst-base-image $(KEXEC_TOOLS_RPM) $(KEXEC_TOOLS_TEST_SRC) $(EXTRA_RPMS) - @echo "Building test base image" - mkdir -p $(TEST_ROOT)/output - $(TEST_ROOT)/scripts/build-image.sh \ - $(BUILD_ROOT)/inst-base-image \ - $(TEST_ROOT)/output/test-base-image \ - $(TEST_ROOT)/scripts/build-scripts/test-base-image.sh \ - $(KEXEC_TOOLS_RPM) $(EXTRA_RPMS) - -test-run: $(TEST_ROOT)/output/test-base-image -ifeq ($(strip $(TEST_CASE)),) - $(TEST_ROOT)/scripts/run-test.sh -else - $(TEST_ROOT)/scripts/run-test.sh --console $(TEST_CASE) -endif - -clean: - rm -rf $(TEST_ROOT)/build - rm -rf $(TEST_ROOT)/output diff --git a/tests/README b/tests/README deleted file mode 100644 index 6ab63a89..00000000 --- a/tests/README +++ /dev/null @@ -1,65 +0,0 @@ -===================== -Kexec Kdump Self-test -===================== - - -Introduction -============ -The self-tests here are useful for quick sanity tests for new patches, and also helpful for debugging issues. - - -How it works -============ -All tests are run within VMs using qemu. By default, VM images are based on Fedora Cloud image, and the image for each test run is a layered qcow2 snapshot on top of the base image. -Test images are managed by Makefile, so if there are any code change in the kexec-tools repository, `make` command will detect that and only rebuild the top image layer. This makes the test runs boot fast and each test run is clean. - - -Basic usage -=========== -Before you start, you can make the self-tests use your own base image by running following command: - -`make clean && make BASE_IMAGE=` - -This is helpful if you have a slow network, else self-test will try to download the cloud image from Fedora's official website using `wget`. - -- Use the following command to run all tests: - $ make test-run - - All available tests will be executed. - - Test artifacts are stored in output/ - -- For easier debugging, you can run only on test with the following command: - $ make TEST_CASE= test-run - - This way, VM's console is directly connected to stdin/out so debugging will be easier. - If there are multiple VMs used in a test case, the VM performing actual kdump/kexec operation will be connected to stdin/out. - -Test Cases -========== -Each test case is a folder under scripts/testcases/, a test case folder will contain at least one executable shell script, and each script should contain two functions: "on_build" and "on_test". - -"on_build" is called when building the test image, which can instruct the self-test framework to install packages or create files, etc. -"on_test" is called when VM finished booting, which can get the boot count by calling "get_test_boot_count" and determine what to do. It should call "test_passed" on success, and call "test_failed" on failure. "test_aborted" is called when unexpected behavior occurs. - -When there are multiple scripts in a single test case folder, they will spawn VMs in lexical order, and the last VM is considered the VM performing the actual test. Other VMs could be hosting test required service. This is useful for the network dump test. However, "test_passed" or "test_failed" or "test_aborted" could be called in any of these VMs, so during network kdump test, the dump target can also terminate the test and mark it passed when a valid vmcore is detected. - - -Debugging -========= - -- When the test VM boots, you can append "no_test" to kernel args in grub, which tells the test services to quit early. - -- You can launch the VMs manually or inspect the image after ran a test. - - Test images are located as: - - output//.img - - Test images' corresponding qemu command are located as: - - output//.qemu_cmd - - To repeat/debug a test manually, you should launch all VMs in output/ menu in lexical order, and append 'no_test' in the last VM's grub cmdline, then VM will hang on login prompt, login with root/fedora. Test script is located as /kexec-kdump-test/test.sh - -- If you just want to inspect the images file content, you can also use scripts/spawn-image-shell.sh to spawn a shell in the image quickly. diff --git a/tests/scripts/build-image.sh b/tests/scripts/build-image.sh deleted file mode 100755 index c196bfb5..00000000 --- a/tests/scripts/build-image.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash - -if [ $# -lt 2 ]; then - echo "Usage: $(basename $0) [] - Build a new on top of , and install - contents defined in . are directly passed - to . - - If is raw, will copy it and create - in qcow2 format. - - If is qcow2, will create as a snapshot - on top of " - exit 1 -fi - -BASEDIR=$(realpath $(dirname "$0")) -. $BASEDIR/image-init-lib.sh - -# Base image to build from -BASE_IMAGE=$1 && shift -if [[ ! -e $BASE_IMAGE ]]; then - perror_exit "Base image '$BASE_IMAGE' not found" -else - BASE_IMAGE=$(realpath "$BASE_IMAGE") -fi - -OUTPUT_IMAGE=$1 && shift -if [[ ! -d $(dirname $OUTPUT_IMAGE) ]]; then - perror_exit "Path '$(dirname $OUTPUT_IMAGE)' doesn't exists" -fi - -INST_SCRIPT=$1 && shift - -create_image_from_base_image $BASE_IMAGE $OUTPUT_IMAGE.building - -mount_image $OUTPUT_IMAGE.building - -img_inst() { - inst_in_image $OUTPUT_IMAGE.building $@ -} - -img_inst_pkg() { - inst_pkg_in_image $OUTPUT_IMAGE.building $@ -} - -img_run_cmd() { - run_in_image $OUTPUT_IMAGE.building "$@" -} - -img_add_qemu_cmd() { - QEMU_CMD+="$@" -} - -[ -e "$INST_SCRIPT" ] && source $INST_SCRIPT - -mv $OUTPUT_IMAGE.building $OUTPUT_IMAGE diff --git a/tests/scripts/build-scripts/base-image.sh b/tests/scripts/build-scripts/base-image.sh deleted file mode 100755 index 1d64cfb5..00000000 --- a/tests/scripts/build-scripts/base-image.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -img_inst_pkg grubby\ - dnsmasq\ - openssh openssh-server\ - dracut-network dracut-squash squashfs-tools ethtool snappy kernel-modules - -img_run_cmd "grubby --args systemd.journald.forward_to_console=1 systemd.log_target=console --update-kernel ALL" -img_run_cmd "grubby --args selinux=0 --update-kernel ALL" diff --git a/tests/scripts/build-scripts/test-base-image.sh b/tests/scripts/build-scripts/test-base-image.sh deleted file mode 100755 index afe1a974..00000000 --- a/tests/scripts/build-scripts/test-base-image.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -# Test RPMs to be installed -TEST_RPMS= -for _rpm in $@; do - if [[ ! -e $_rpm ]]; then - perror_exit "'$_rpm' not found" - else - TEST_RPMS="$TEST_RPMS $(realpath "$_rpm")" - fi -done - -img_run_cmd "mkdir -p /kexec-kdump-test" -img_inst $TESTDIR/scripts/kexec-kdump-test/init.sh /kexec-kdump-test/init.sh -img_inst $TESTDIR/scripts/kexec-kdump-test/test.sh /kexec-kdump-test/test.sh -img_inst $TESTDIR/scripts/kexec-kdump-test/kexec-kdump-test.service /etc/systemd/system/kexec-kdump-test.service -img_run_cmd "systemctl enable kexec-kdump-test.service" - -img_inst_pkg $TEST_RPMS -# Test script should start kdump manually to save time -img_run_cmd "systemctl disable kdump.service" diff --git a/tests/scripts/build-scripts/test-image.sh b/tests/scripts/build-scripts/test-image.sh deleted file mode 100755 index d8e907ed..00000000 --- a/tests/scripts/build-scripts/test-image.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -. $TESTDIR/scripts/test-lib.sh -TEST_SCRIPT=$1 - -QEMU_CMD="$DEFAULT_QEMU_CMD \ --serial stdio \ --serial file:$(get_test_output_file $TEST_SCRIPT) \ --monitor none \ --hda $OUTPUT_IMAGE" - -img_add_qemu_cmd() { - QEMU_CMD+=" $@" -} - -source $TEST_SCRIPT - -on_build - -img_inst $TEST_SCRIPT /kexec-kdump-test/test.sh - -echo $QEMU_CMD > $(get_test_qemu_cmd_file $TEST_SCRIPT) diff --git a/tests/scripts/copy-from-image.sh b/tests/scripts/copy-from-image.sh deleted file mode 100755 index 9b71054c..00000000 --- a/tests/scripts/copy-from-image.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -BASEDIR=$(realpath $(dirname "$0")) -. $BASEDIR/image-init-lib.sh - -# Base image to copy from -BOOT_IMAGE=$1 && shift -if [ ! -e "$BOOT_IMAGE" ]; then - perror_exit "Image '$BOOT_IMAGE' not found" -else - BOOT_IMAGE=$(realpath "$BOOT_IMAGE") -fi - -mount_image $BOOT_IMAGE - -IMAGE_MNT=$(get_image_mount_root $BOOT_IMAGE) - -SRC= -while [ $# -gt 1 ]; do - SRC="$SRC $IMAGE_MNT/$1" - shift -done -DST=$1 - -cp -rv $SRC $DST diff --git a/tests/scripts/image-init-lib.sh b/tests/scripts/image-init-lib.sh deleted file mode 100644 index b0af7f8c..00000000 --- a/tests/scripts/image-init-lib.sh +++ /dev/null @@ -1,275 +0,0 @@ -#!/usr/bin/env bash -[ -z "$TESTDIR" ] && TESTDIR=$(realpath $(dirname "$0")/../) - -SUDO="sudo" - -declare -A MNTS=() -declare -A DEVS=() - -perror() { - echo $@>&2 -} - -perror_exit() { - echo $@>&2 - exit 1 -} - -is_mounted() -{ - findmnt -k -n $1 &>/dev/null -} - -clean_up() -{ - for _mnt in ${MNTS[@]}; do - is_mounted $_mnt && $SUDO umount -f -R $_mnt - done - - for _dev in ${DEVS[@]}; do - [ ! -e "$_dev" ] && continue - [[ "$_dev" == "/dev/loop"* ]] && $SUDO losetup -d "$_dev" - [[ "$_dev" == "/dev/nbd"* ]] && $SUDO qemu-nbd --disconnect "$_dev" - done - - [ -d "$TMPDIR" ] && $SUDO rm --one-file-system -rf -- "$TMPDIR"; - - sync -} - -trap ' -ret=$?; -clean_up -exit $ret; -' EXIT - -# clean up after ourselves no matter how we die. -trap 'exit 1;' SIGINT - -readonly TMPDIR="$(mktemp -d -t kexec-kdump-test.XXXXXX)" -[ -d "$TMPDIR" ] || perror_exit "mktemp failed." - -get_image_fmt() { - local image=$1 fmt - - [ ! -e "$image" ] && perror "image: $image doesn't exist" && return 1 - - fmt=$(qemu-img info $image | sed -n "s/file format:\s*\(.*\)/\1/p") - - [ $? -eq 0 ] && echo $fmt && return 0 - - return 1 -} - -fmt_is_qcow2() { - [ "$1" == "qcow2" ] || [ "$1" == "qcow2 backing qcow2" ] -} - -# If it's partitioned, return the mountable partition, else return the dev -get_mountable_dev() { - local dev=$1 parts - - $SUDO partprobe $dev && sync - parts="$(ls -1 ${dev}p*)" - if [ -n "$parts" ]; then - if [ $(echo "$parts" | wc -l) -gt 1 ]; then - perror "It's a image with multiple partitions, using last partition as main partition" - fi - echo "$parts" | tail -1 - else - echo "$dev" - fi -} - -# get the separate boot partition -# return the 2nd partition as boot partition -get_mount_boot() { - local dev=$1 _second_part=${dev}p2 - - # it's better to check if the 2nd partition has the boot label using lsblk - # but somehow this doesn't work starting with Fedora37 - echo $_second_part -} - - -prepare_loop() { - [ -n "$(lsmod | grep "^loop")" ] && return - - $SUDO modprobe loop - - [ ! -e "/dev/loop-control" ] && perror_exit "failed to load loop driver" -} - -prepare_nbd() { - [ -n "$(lsmod | grep "^nbd")" ] && return - - $SUDO modprobe nbd max_part=4 - - [ ! -e "/dev/nbd0" ] && perror_exit "failed to load nbd driver" -} - -mount_nbd() { - local image=$1 size dev - for _dev in /sys/class/block/nbd* ; do - size=$(cat $_dev/size) - if [ "$size" -eq 0 ] ; then - dev=/dev/${_dev##*/} - $SUDO qemu-nbd --connect=$dev $image 1>&2 - [ $? -eq 0 ] && echo $dev && break - fi - done - - return 1 -} - -image_lock() -{ - local image=$1 timeout=5 fd - - eval "exec {fd}>$image.lock" - if [ $? -ne 0 ]; then - perror_exit "failed acquiring image lock" - exit 1 - fi - - flock -n $fd - rc=$? - while [ $rc -ne 0 ]; do - echo "Another instance is holding the image lock ..." - flock -w $timeout $fd - rc=$? - done -} - -# Mount a device, will umount it automatially when shell exits -mount_image() { - local image=$1 fmt - local dev mnt mnt_dev boot root - - # Lock the image just in case user run this script in parrel - image_lock $image - - fmt=$(get_image_fmt $image) - [ $? -ne 0 ] || [ -z "$fmt" ] && perror_exit "failed to detect image format" - - if [ "$fmt" == "raw" ]; then - prepare_loop - - dev="$($SUDO losetup --show -f $image)" - [ $? -ne 0 ] || [ -z "$dev" ] && perror_exit "failed to setup loop device" - - elif fmt_is_qcow2 "$fmt"; then - prepare_nbd - - dev=$(mount_nbd $image) - [ $? -ne 0 ] || [ -z "$dev" ] perror_exit "failed to connect qemu to nbd device '$dev'" - else - perror_exit "Unrecognized image format '$fmt'" - fi - DEVS[$image]="$dev" - - mnt="$(mktemp -d -p $TMPDIR -t mount.XXXXXX)" - [ $? -ne 0 ] || [ -z "$mnt" ] && perror_exit "failed to create tmp mount dir" - MNTS[$image]="$mnt" - - mnt_dev=$(get_mountable_dev "$dev") - [ $? -ne 0 ] || [ -z "$mnt_dev" ] && perror_exit "failed to setup loop device" - - $SUDO mount $mnt_dev $mnt - [ $? -ne 0 ] && perror_exit "failed to mount device '$mnt_dev'" - boot=$(get_mount_boot "$dev") - if [[ -n "$boot" ]]; then - root=$(get_image_mount_root $image) - $SUDO mount $boot $root/boot - [ $? -ne 0 ] && perror_exit "failed to mount the bootable partition for device '$mnt_dev'" - fi -} - -get_image_mount_root() { - local image=$1 - local root=${MNTS[$image]} - - # Starting from Fedora 36, the root node is /root/root of the last partition - [ -d "$root/root/root" ] && root=$root/root - echo $root - - if [ -z "$root" ]; then - return 1 - fi -} - -shell_in_image() { - local root=$(get_image_mount_root $1) && shift - - pushd $root - - $SHELL - - popd -} - -inst_pkg_in_image() { - local root=$(get_image_mount_root $1) && shift - - # LSB not available - # release_info=$($SUDO chroot $root /bin/bash -c "lsb_release -a") - # release=$(echo "$release_info" | sed -n "s/Release:\s*\(.*\)/\1/p") - # distro=$(echo "$release_info" | sed -n "s/Distributor ID:\s*\(.*\)/\1/p") - # if [ "$distro" != "Fedora" ]; then - # perror_exit "only Fedora image is supported" - # fi - release=$(cat $root/etc/fedora-release | sed -n "s/.*[Rr]elease\s*\([0-9]*\).*/\1/p") - [ $? -ne 0 ] || [ -z "$release" ] && perror_exit "only Fedora image is supported" - - $SUDO dnf --releasever=$release --installroot=$root install -y $@ -} - -run_in_image() { - local root=$(get_image_mount_root $1) && shift - - $SUDO chroot $root /bin/bash -c "$@" -} - -inst_in_image() { - local image=$1 src=$2 dst=$3 - local root=$(get_image_mount_root $1) - - $SUDO cp $src $root/$dst -} - -# If source image is qcow2, create a snapshot -# If source image is raw, convert to raw -# If source image is xz, decompress then repeat the above logic -# -# Won't touch source image -create_image_from_base_image() { - local image=$1 - local output=$2 - local decompressed_image - - local ext="${image##*.}" - if [[ "$ext" == 'xz' ]]; then - echo "Decompressing base image..." - xz -d -k $image - decompressed_image=${image%.xz} - image=$decompressed_image - fi - - local image_fmt=$(get_image_fmt $image) - if [ "$image_fmt" != "raw" ]; then - if fmt_is_qcow2 "$image_fmt"; then - echo "Source image is qcow2, using snapshot..." - qemu-img create -f qcow2 -b $image -F qcow2 $output - else - perror_exit "Unrecognized base image format '$image_mnt'" - fi - else - echo "Source image is raw, converting to qcow2..." - qemu-img convert -f raw -O qcow2 $image $output - fi - - # Clean up decompress temp image - if [ -n "$decompressed_image" ]; then - rm $decompressed_image - fi -} diff --git a/tests/scripts/kexec-kdump-test/init.sh b/tests/scripts/kexec-kdump-test/init.sh deleted file mode 100755 index deacb51c..00000000 --- a/tests/scripts/kexec-kdump-test/init.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env sh -BOOT_ARG="test_boot_count" -_YELLOW='\033[1;33m' -_GREEN='\033[0;32m' -_RED='\033[0;31m' -_NC='\033[0m' # No Color - -if [ -n "$(cat /proc/cmdline | grep "\bno_test\b")" ]; then - exit 0 -fi - -get_test_boot_count() { - local boot_count=$(cat /proc/cmdline | sed -n "s/.*$BOOT_ARG=\([0-9]*\).*/\1/p") - - if [ -z "$boot_count" ]; then - boot_count=1 - fi - - echo $boot_count -} - -test_output() { - echo $@ > /dev/ttyS1 - echo $@ > /dev/ttyS0 - - sync -} - -test_passed() { - echo -e "${_GREEN}TEST PASSED${_NC}" > /dev/ttyS1 - echo -e "${_GREEN}kexec-kdump-test: TEST PASSED${_NC}" > /dev/ttyS0 - - echo $@ > /dev/ttyS1 - echo $@ > /dev/ttyS0 - - sync - - shutdown -h 0 - - exit 0 -} - -test_failed() { - echo -e "${_RED}TEST FAILED${_NC}" > /dev/ttyS1 - echo -e "${_RED}kexec-kdump-test: TEST FAILED${_NC}" > /dev/ttyS0 - - echo $@ > /dev/ttyS1 - echo $@ > /dev/ttyS0 - - sync - - shutdown -h 0 - - exit 1 -} - -test_abort() { - echo -e "${_YELLOW}TEST ABORTED${_NC}" > /dev/ttyS1 - echo -e "${_YELLOW}kexec-kdump-test: TEST ABORTED${_NC}" > /dev/ttyS0 - - echo $@ > /dev/ttyS1 - echo $@ > /dev/ttyS0 - - sync - - shutdown -h 0 - - exit 2 -} - -has_valid_vmcore_dir() { - local path=$1 - local vmcore_dir=$path/$(ls -1 $path | tail -n 1) - local vmcore="" - - test_output "Found a vmcore dir \"$vmcore_dir\":" - # Checking with `crash` is slow and consume a lot of memory/disk, - # just do a sanity check by check if log are available. - if [ -e $vmcore_dir/vmcore ]; then - vmcore=$vmcore_dir/vmcore - makedumpfile --dump-dmesg $vmcore $vmcore_dir/vmcore-dmesg.txt.2 || { - test_output "Failed to retrive dmesg from vmcore!" - return 1 - } - elif [ -e $vmcore_dir/vmcore.flat ]; then - vmcore=$vmcore_dir/vmcore.flat - makedumpfile -R $vmcore_dir/vmcore < $vmcore || return 1 - makedumpfile --dump-dmesg $vmcore_dir/vmcore $vmcore_dir/vmcore-dmesg.txt.2 || { - test_output "Failed to retrive dmesg from vmcore!" - return 1 - } - rm $vmcore_dir/vmcore - else - test_output "The vmcore dir is empty!" - return 1 - fi - - if ! diff -w $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then - test_output "Dmesg retrived from vmcore is different from dump version!" - return 1 - fi - - test_output "VMCORE: $vmcore" - test_output "KERNEL VERSION: $(rpm -q kernel-core)" - - return 0 -} - -BOOT_COUNT=$(get_test_boot_count) -test_output "Kexec-Kdump-Test Boot #$BOOT_COUNT" - -echo 'fedora' | passwd --stdin root - -test_output "Updating kernel cmdline" -grubby --update-kernel ALL --args $BOOT_ARG=$(expr $BOOT_COUNT + 1) && sync - -test_output "Executing test hook" -source /kexec-kdump-test/test.sh - -on_test; - -test_output "Test exited, system hang for inspect" diff --git a/tests/scripts/kexec-kdump-test/kexec-kdump-test.service b/tests/scripts/kexec-kdump-test/kexec-kdump-test.service deleted file mode 100644 index ba7b11e5..00000000 --- a/tests/scripts/kexec-kdump-test/kexec-kdump-test.service +++ /dev/null @@ -1,9 +0,0 @@ -[Unit] -Description=Kexec Kdump Test Service - -[Service] -ExecStart=/kexec-kdump-test/init.sh -Type=idle - -[Install] -WantedBy=multi-user.target diff --git a/tests/scripts/kexec-kdump-test/test.sh b/tests/scripts/kexec-kdump-test/test.sh deleted file mode 100755 index 66ac15d7..00000000 --- a/tests/scripts/kexec-kdump-test/test.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env sh -# A test example that do nothing - -# Executed before VM starts -on_build() { - : -} - -# Executed when VM boots -on_test() { - : - # call get_test_boot_count to get boot cound - # call test_passed if test passed - # call test_failed if test passed -} diff --git a/tests/scripts/run-qemu b/tests/scripts/run-qemu deleted file mode 100755 index 836387af..00000000 --- a/tests/scripts/run-qemu +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Check which virtualization technology to use -# We prefer kvm, kqemu, userspace in that order. - -# This script is based on https://github.com/dracutdevs/dracut/blob/master/test/run-qemu - -export PATH=/sbin:/bin:/usr/sbin:/usr/bin - -[[ -x /usr/bin/qemu ]] && BIN=/usr/bin/qemu && ARGS="-cpu max" -$(lsmod | grep -q '^kqemu ') && BIN=/usr/bin/qemu && ARGS="-kernel-kqemu -cpu host" -[[ -c /dev/kvm && -x /usr/bin/kvm ]] && BIN=/usr/bin/kvm && ARGS="-cpu host" -[[ -c /dev/kvm && -x /usr/bin/qemu-kvm ]] && BIN=/usr/bin/qemu-kvm && ARGS="-cpu host" -[[ -c /dev/kvm && -x /usr/libexec/qemu-kvm ]] && BIN=/usr/libexec/qemu-kvm && ARGS="-cpu host" -[[ -x /usr/bin/qemu-system-$(uname -i) ]] && BIN=/usr/bin/qemu-system-$(uname -i) && ARGS="-cpu max" -[[ -c /dev/kvm && -x /usr/bin/qemu-system-$(uname -i) ]] && BIN=/usr/bin/qemu-system-$(uname -i) && ARGS="-enable-kvm -cpu host" - -[[ $BIN ]] || { - echo "Could not find a working KVM or QEMU to test with!" >&2 - echo "Please install kvm or qemu." >&2 - exit 1 -} - -exec $BIN $ARGS "$@" diff --git a/tests/scripts/run-test.sh b/tests/scripts/run-test.sh deleted file mode 100755 index 0b589f70..00000000 --- a/tests/scripts/run-test.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/bin/bash - -_kill_if_valid_pid() { - local _pid="$1" - if ps -p "$_pid" > /dev/null - then - kill "$_pid" - fi -} - -_recursive_kill() { - local _pid="$1" - local _children _child - - _children=$(pgrep -P "$_pid") - if [ -n "$_children" ]; then - for _child in $_children - do - _recursive_kill "$_child" - _kill_if_valid_pid "$_child" - done - fi - _kill_if_valid_pid "$_pid" -} - -_kill_all_jobs() { - local _jobs=$(jobs -r -p) - local _job - - if [ -n "$_jobs" ]; then - for _job in $_jobs - do - _recursive_kill "$_job" - done - fi -} - -trap ' -ret=$?; -_kill_all_jobs -exit $ret; -' EXIT - -trap 'exit 1;' SIGINT - -BASEDIR=$(realpath $(dirname "$0")) -. $BASEDIR/test-lib.sh -TESTCASEDIR="$BASEDIR/testcases" - -console=0 -testcases="" - -while [ $# -gt 0 ]; do - case $1 in - '') - break - ;; - --console ) - console=1 - ;; - -*) - echo "Invalid option $1" - ;; - *) - testcases+=" $1" - ;; - esac - shift; -done - -if [ -z "$testcases" ]; then - echo "==== Starting all tests: ====" - testcases=$(ls -1 $TESTCASEDIR) -else - echo "==== Starting specified tests: ====" -fi -echo ${testcases##*/} -echo - -declare -A results -ret=0 - -for test_case in $testcases; do - echo "======== Running Test Case $test_case ========" - results[$test_case]="" - - testdir=$TESTCASEDIR/$test_case - scripts=$(ls -r -1 $testdir | egrep "\.sh$" | tr '\n' ' ') - test_outputs="" - read main_script aux_script <<< "$scripts" - - if [ -z "$main_script" ]; then - echo "ERROR: Empty testcase dir $testdir" - continue - fi - - for script in $scripts; do - echo "---- Building image for: $script ----" - echo "-------- Output image is: $(get_test_image $testdir/$script)" - echo "-------- Building log is: $(get_test_image $testdir/$script).log" - - mkdir -p $(dirname $(get_test_image $testdir/$script)) - build_test_image $testdir/$script &> $(get_test_image $testdir/$script).log - - if [ $? -ne 0 ]; then - echo "Failing building image!" - continue 2 - fi - done - - for script in $aux_script; do - echo "---- Starting VM: $script ----" - - script="$testdir/$script" - echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)" - echo "-------- Console log: $(get_test_console_file $script)" - echo "-------- Test log: $(get_test_output_file $script)" - test_outputs+="$(get_test_output_file $script) " - - rm -f $(get_test_console_file $script) - rm -f $(get_test_output_file $script) - - $(run_test_sync $script > $(get_test_console_file $script)) & - - sleep 5 - done - - script="$main_script" - echo "---- Starting test VM: $(basename $script) ----" - script="$testdir/$script" - - echo "-------- Qemu cmdline: $(get_test_qemu_cmd_file $script)" - echo "-------- Console log: $(get_test_console_file $script)" - echo "-------- Test log: $(get_test_output_file $script)" - test_outputs+="$(get_test_output_file $script) " - - rm -f $(get_test_console_file $script) - rm -f $(get_test_output_file $script) - - if [ $console -eq 1 ]; then - run_test_sync $script | tee $(get_test_console_file $script) - [ -n "$(jobs -p)" ] && wait $(jobs -p) - else - $(run_test_sync $script > $(get_test_console_file $script)) & - watch_test_outputs $test_outputs - fi - - res="$(gather_test_result $test_outputs)" - - [ $? -ne 0 ] && ret=$(expr $ret + 1) - results[$test_case]="$res" - _kill_all_jobs - echo -e "-------- Test finished: $test_case $res --------" - for script in $scripts; do - script="$testdir/$script" - output="$(get_test_output_file $script) " - image="$(get_test_image $script)" - vmcore="$(sed -n 's/^VMCORE: \(\S*\).*/\1/p' $output)" - kernel="$(sed -n 's/^KERNEL VERSION: \(\S*\).*/\1/p' $output)" - if [ -n "$vmcore" ]; then - echo "You can retrive the verify the vmcore file using following command:" - echo "./scripts/copy-from-image.sh \\" - echo " $image \\" - echo " $vmcore ./" - echo "Kernel package verion is: $kernel" - fi - done -done - -echo "======== Test results ========" -for i in ${!results[@]}; do - echo "----------------" - echo -e "$i:\t\t${results[$i]}" -done - -exit $ret diff --git a/tests/scripts/spawn-image-shell.sh b/tests/scripts/spawn-image-shell.sh deleted file mode 100755 index ccfb655f..00000000 --- a/tests/scripts/spawn-image-shell.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash - -BASEDIR=$(realpath $(dirname "$0")) -. $BASEDIR/image-init-lib.sh - -# Base image to build from -BOOT_IMAGE=$1 -if [[ ! -e $BOOT_IMAGE ]]; then - perror_exit "Image '$BOOT_IMAGE' not found" -else - BOOT_IMAGE=$(realpath "$BOOT_IMAGE") -fi - -mount_image $BOOT_IMAGE - -shell_in_image $BOOT_IMAGE diff --git a/tests/scripts/test-lib.sh b/tests/scripts/test-lib.sh deleted file mode 100644 index 57b05d37..00000000 --- a/tests/scripts/test-lib.sh +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env sh -[ -z "$BASEDIR" ] && BASEDIR=$(realpath $(dirname "$0")) -[ -z "$TESTDIR" ] && TESTDIR=$(realpath $BASEDIR/../) -[ -z "$TEST_BASE_IMAGE" ] && TEST_BASE_IMAGE=$TESTDIR/output/test-base-image - -[[ ! -e $TEST_BASE_IMAGE ]] && echo "Test base image not found." && exit 1 - -DEFAULT_QEMU_CMD="-nodefaults \ --nographic \ --smp 2 \ --m 1G \ --monitor none" - -_YELLOW='\033[1;33m' -_GREEN='\033[0;32m' -_RED='\033[0;31m' -_NC='\033[0m' # No Color - -get_test_path() { - local script=$1 - local testname=$(basename $(dirname $script)) - local output=$TESTDIR/output/$testname - - echo $output -} - -get_test_entry_name() { - echo $(basename ${1%.*}) -} - -get_test_image() { - local script=$1 - local testout=$(get_test_path $script) - local entry=$(get_test_entry_name $script) - - echo $testout/$entry.img -} - -get_test_qemu_cmd_file() { - local script=$1 - local testout=$(get_test_path $script) - local entry=$(get_test_entry_name $script) - - echo $testout/$entry.qemu_cmd -} - -get_test_qemu_cmd() { - cat $(get_test_qemu_cmd_file $1) -} - -get_test_output_file() { - local script=$1 - local testout=$(get_test_path $script) - local entry=$(get_test_entry_name $script) - - echo $testout/$entry.output -} - -get_test_console_file() { - local script=$1 - local testout=$(get_test_path $script) - local entry=$(get_test_entry_name $script) - - echo $testout/$entry.console -} - -get_test_output() { - local output=$(get_test_output_file $1) - if [ -e "$output" ]; then - cat $(get_test_output_file $1) - else - echo "" - fi -} - -build_test_image() { - local script=$1 - local test_image=$(get_test_image $script) - mkdir -p $(dirname $test_image) - - $BASEDIR/build-image.sh \ - $TEST_BASE_IMAGE \ - $test_image \ - $BASEDIR/build-scripts/test-image.sh \ - $script -} - -run_test_sync() { - local qemu_cmd=$(get_test_qemu_cmd $1) - - if [ -n "$qemu_cmd" ]; then - timeout --foreground 10m $BASEDIR/run-qemu $(get_test_qemu_cmd $1) - else - echo "error: test qemu command line is not configured" > /dev/stderr - return 1 - fi -} - -_check_test_result() { - grep "TEST PASSED" $1 2>/dev/null - [ $? -eq 0 ] && return 0 - - grep "TEST FAILED" $1 2>/dev/null - [ $? -eq 0 ] && return 1 - - grep "TEST ABORTED" $1 2>/dev/null - [ $? -eq 0 ] && return 2 - - return 255 -} - -# Print test result and return below value: -# 0: Test passed -# 1: Test failed -# 2: Test aborted, test scripts errored out -# 3: Test exited unexpectely, VM got killed early, or time out -gather_test_result() { - local ret=255 - local res="" - - for i in $@; do - res=$(_check_test_result $i) - ret=$? - - if [ $ret -ne 255 ]; then - echo $res - return $ret - fi - done - - echo "${_RED}TEST RESULT NOT FOUND!${_NC}" - return 3 -} - -# Wait and watch for test result -watch_test_outputs() { - local ret=255 - local res="" - # If VMs are still running, check for test result, if - # test finished, kill remaining VMs - while true; do - if [ -n "$(jobs -r)" ]; then - # VMs still running - for i in $@; do - res=$(_check_test_result $i) - ret=$? - - if [ $ret -ne 255 ]; then - # Test finished - break 2 - fi - done - else - # VMs exited - ret=255 - - for i in $@; do - res=$(_check_test_result $i) - ret=$? - - if [ $ret -ne 255 ]; then - break 2 - fi - done - - if [ $ret -eq 255 ]; then - ret=3 - break - fi - fi - - sleep 1 - done - - return $ret -} diff --git a/tests/scripts/testcases/local-kdump/0-local.sh b/tests/scripts/testcases/local-kdump/0-local.sh deleted file mode 100755 index e5c1ff01..00000000 --- a/tests/scripts/testcases/local-kdump/0-local.sh +++ /dev/null @@ -1,32 +0,0 @@ -on_build() { - : -} - -on_test() { - local boot_count=$(get_test_boot_count) - - if [ $boot_count -eq 1 ]; then - cat << EOF > /etc/kdump.conf -path /var/crash -core_collector makedumpfile -l --message-level 7 -d 31 -EOF - kdumpctl start || test_failed "Failed to start kdump" - - sync - - echo 1 > /proc/sys/kernel/sysrq - echo c > /proc/sysrq-trigger - - elif [ $boot_count -eq 2 ]; then - - if has_valid_vmcore_dir /var/crash; then - test_passed - else - test_failed "Vmcore missing" - fi - - shutdown -h 0 - else - test_failed "Unexpected reboot" - fi -} diff --git a/tests/scripts/testcases/lvm2-thinp-kdump/0-local-lvm2-thinp.sh b/tests/scripts/testcases/lvm2-thinp-kdump/0-local-lvm2-thinp.sh deleted file mode 100755 index 5470aa06..00000000 --- a/tests/scripts/testcases/lvm2-thinp-kdump/0-local-lvm2-thinp.sh +++ /dev/null @@ -1,59 +0,0 @@ -on_build() { - TEST_DIR_PREFIX=/tmp/lvm_test.XXXXXX - # clear TEST_DIRs if any - rm -rf ${TEST_DIR_PREFIX%.*}.* - TEST_IMG="$(mktemp -d $TEST_DIR_PREFIX)/test.img" - - img_inst_pkg "lvm2" - img_inst $TESTDIR/scripts/testcases/lvm2-thinp-kdump/lvm.conf /etc/lvm/ - dd if=/dev/zero of=$TEST_IMG bs=300M count=1 - # The test.img will be /dev/sdb - img_add_qemu_cmd "-hdb $TEST_IMG" -} - -on_test() { - VG=vg00 - LV_THINPOOL=thinpool - LV_VOLUME=thinlv - VMCORE_PATH=var/crash - - local boot_count=$(get_test_boot_count) - - if [ $boot_count -eq 1 ]; then - - vgcreate $VG /dev/sdb - # Create a small thinpool which is definitely not enough for - # vmcore, then create a thin volume which is definitely enough - # for vmcore, so we can make sure thinpool should be autoextend - # during runtime. - lvcreate -L 10M -T $VG/$LV_THINPOOL - lvcreate -V 300M -T $VG/$LV_THINPOOL -n $LV_VOLUME - mkfs.ext4 /dev/$VG/$LV_VOLUME - mount /dev/$VG/$LV_VOLUME /mnt - mkdir -p /mnt/$VMCORE_PATH - - cat << EOF > /etc/kdump.conf -ext4 /dev/$VG/$LV_VOLUME -path /$VMCORE_PATH -core_collector makedumpfile -l --message-level 7 -d 31 -EOF - kdumpctl start || test_failed "Failed to start kdump" - - sync - - echo 1 > /proc/sys/kernel/sysrq - echo c > /proc/sysrq-trigger - - elif [ $boot_count -eq 2 ]; then - mount /dev/$VG/$LV_VOLUME /mnt - if has_valid_vmcore_dir /mnt/$VMCORE_PATH; then - test_passed - else - test_failed "Vmcore missing" - fi - - shutdown -h 0 - else - test_failed "Unexpected reboot" - fi -} diff --git a/tests/scripts/testcases/lvm2-thinp-kdump/lvm.conf b/tests/scripts/testcases/lvm2-thinp-kdump/lvm.conf deleted file mode 100644 index 4d81fbd0..00000000 --- a/tests/scripts/testcases/lvm2-thinp-kdump/lvm.conf +++ /dev/null @@ -1,5 +0,0 @@ -activation { - thin_pool_autoextend_threshold = 70 - thin_pool_autoextend_percent = 20 - monitoring = 1 -} \ No newline at end of file diff --git a/tests/scripts/testcases/nfs-early-kdump/0-server.sh b/tests/scripts/testcases/nfs-early-kdump/0-server.sh deleted file mode 120000 index 3f888ce5..00000000 --- a/tests/scripts/testcases/nfs-early-kdump/0-server.sh +++ /dev/null @@ -1 +0,0 @@ -../nfs-kdump/0-server.sh \ No newline at end of file diff --git a/tests/scripts/testcases/nfs-early-kdump/1-client.sh b/tests/scripts/testcases/nfs-early-kdump/1-client.sh deleted file mode 100755 index dfef4d8b..00000000 --- a/tests/scripts/testcases/nfs-early-kdump/1-client.sh +++ /dev/null @@ -1,46 +0,0 @@ -# Executed before VM starts -on_build() { - img_inst_pkg "nfs-utils" - img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57" -} - -on_test() { - local boot_count=$(get_test_boot_count) - local nfs_server=192.168.77.1 - local earlykdump_path="/usr/lib/dracut/modules.d/99earlykdump/early-kdump.sh" - local tmp_file="/tmp/.tmp-file" - - if [[ ! -f $earlykdump_path ]]; then - test_failed "early-kdump.sh not exist!" - fi - - if [ $boot_count -eq 1 ]; then - cat << EOF > /etc/kdump.conf -nfs $nfs_server:/srv/nfs -core_collector makedumpfile -l --message-level 7 -d 31 -final_action poweroff -EOF - - while ! ping -c 1 $nfs_server -W 1; do - sleep 1 - done - - kdumpctl start \ - || test_failed "Failed to start kdump" - grubby --update-kernel=ALL --args=rd.earlykdump - - cat << EOF > $tmp_file -echo 1 > /proc/sys/kernel/sysrq -echo c > /proc/sysrq-trigger -EOF - sed -i "/early_kdump_load$/r $tmp_file" $earlykdump_path - dracut -f --add earlykdump - kdumpctl restart \ - || test_failed "Failed to start earlykdump" - - sync - reboot - else - test_failed "Unexpected reboot" - fi -} diff --git a/tests/scripts/testcases/nfs-kdump/0-server.sh b/tests/scripts/testcases/nfs-kdump/0-server.sh deleted file mode 100755 index 5436d3d2..00000000 --- a/tests/scripts/testcases/nfs-kdump/0-server.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env sh - -# Executed before VM starts -on_build() { - img_inst_pkg "nfs-utils dnsmasq" - - img_run_cmd "mkdir -p /srv/nfs/var/crash" - img_run_cmd "echo /srv/nfs 192.168.77.1/24\(rw,async,insecure,no_root_squash\) > /etc/exports" - img_run_cmd "systemctl enable nfs-server" - - img_run_cmd "touch /etc/systemd/resolved.conf" - img_run_cmd "echo DNSStubListener=no >> /etc/systemd/resolved.conf" - - img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf" - img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf" - img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h >> /etc/dnsmasq.conf" - img_run_cmd "systemctl enable dnsmasq" - - img_run_cmd 'echo [connection] > /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo type=ethernet >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo interface-name=eth0 >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo [ipv4] >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo address1=192.168.77.1/24 >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo method=manual >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'chmod 600 /etc/NetworkManager/system-connections/eth0.nmconnection' - - img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56" -} - -# Executed when VM boots -on_test() { - while true; do - if has_valid_vmcore_dir /srv/nfs/var/crash; then - # Wait a few seconds so client finish it's work to generate a full log - sleep 5 - - test_passed - fi - - sleep 1 - done -} diff --git a/tests/scripts/testcases/nfs-kdump/1-client.sh b/tests/scripts/testcases/nfs-kdump/1-client.sh deleted file mode 100755 index 184c7955..00000000 --- a/tests/scripts/testcases/nfs-kdump/1-client.sh +++ /dev/null @@ -1,30 +0,0 @@ -# Executed before VM starts -on_build() { - img_inst_pkg "nfs-utils" - img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57" -} - -on_test() { - local boot_count=$(get_test_boot_count) - local nfs_server=192.168.77.1 - - if [ "$boot_count" -eq 1 ]; then - cat << EOF > /etc/kdump.conf -nfs $nfs_server:/srv/nfs -core_collector makedumpfile -l --message-level 7 -d 31 -EOF - - while ! ping -c 1 $nfs_server -W 1; do - sleep 1 - done - - kdumpctl start || test_failed "Failed to start kdump" - - sync - - echo 1 > /proc/sys/kernel/sysrq - echo c > /proc/sysrq-trigger - else - shutdown -h 0 - fi -} diff --git a/tests/scripts/testcases/ssh-kdump/0-server.sh b/tests/scripts/testcases/ssh-kdump/0-server.sh deleted file mode 100755 index b4b84c96..00000000 --- a/tests/scripts/testcases/ssh-kdump/0-server.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env sh - -# Executed before VM starts -on_build() { - img_add_qemu_cmd "-nic socket,listen=:8010,mac=52:54:00:12:34:56" - - img_run_cmd "echo root:fedora | chpasswd" - img_run_cmd 'sed -i "s/^.*PasswordAuthentication .*\$/PasswordAuthentication yes/" /etc/ssh/sshd_config' - img_run_cmd 'sed -i "s/^.*PermitRootLogin .*\$/PermitRootLogin yes/" /etc/ssh/sshd_config' - img_run_cmd "systemctl enable sshd" - - img_run_cmd "touch /etc/systemd/resolved.conf" - img_run_cmd "echo DNSStubListener=no >> /etc/systemd/resolved.conf" - - img_run_cmd "echo interface=eth0 > /etc/dnsmasq.conf" - img_run_cmd "echo dhcp-authoritative >> /etc/dnsmasq.conf" - img_run_cmd "echo dhcp-range=192.168.77.50,192.168.77.100,255.255.255.0,12h >> /etc/dnsmasq.conf" - img_run_cmd "systemctl enable dnsmasq" - - img_run_cmd 'echo [connection] > /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo type=ethernet >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo interface-name=eth0 >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo [ipv4] >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo address1=192.168.77.1/24 >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'echo method=manual >> /etc/NetworkManager/system-connections/eth0.nmconnection' - img_run_cmd 'chmod 600 /etc/NetworkManager/system-connections/eth0.nmconnection' -} - -# Executed when VM boots -on_test() { - while true; do - if has_valid_vmcore_dir /var/crash; then - test_passed - fi - - sleep 1 - done -} diff --git a/tests/scripts/testcases/ssh-kdump/1-client.sh b/tests/scripts/testcases/ssh-kdump/1-client.sh deleted file mode 100755 index be549c31..00000000 --- a/tests/scripts/testcases/ssh-kdump/1-client.sh +++ /dev/null @@ -1,42 +0,0 @@ -# Executed before VM starts -on_build() { - img_inst_pkg "sshpass" - img_add_qemu_cmd "-nic socket,connect=127.0.0.1:8010,mac=52:54:00:12:34:57" -} - -on_test() { - local boot_count=$(get_test_boot_count) - local ssh_server=192.168.77.1 - - if [ "$boot_count" -eq 1 ]; then -cat << EOF > /etc/kdump.conf -ssh root@192.168.77.1 -core_collector makedumpfile -l --message-level 7 -d 31 -F -EOF - - ssh-keygen -q -t rsa -N '' -f /root/.ssh/id_rsa <<< y - - while ! ping -c 1 $ssh_server -W 1; do - sleep 1 - done - - while [ -z "$(cat /root/.ssh/known_hosts)" ]; do - ssh-keyscan -H 192.168.77.1 > /root/.ssh/known_hosts - done - - sshpass -p fedora ssh $ssh_server "mkdir /root/.ssh" - cat /root/.ssh/id_rsa.pub | sshpass -p fedora ssh $ssh_server "cat >> /root/.ssh/authorized_keys" - - sshpass -p fedora kdumpctl propagate - cat /root/.ssh/kdump_id_rsa.pub | sshpass -p fedora ssh $ssh_server "cat >> /root/.ssh/authorized_keys" - - kdumpctl start || test_failed "Failed to start kdump" - - sync - - echo 1 > /proc/sys/kernel/sysrq - echo c > /proc/sysrq-trigger - else - shutdown -h 0 - fi -} From 7428f4e1ff2634d56c0f0c0165f5138904d3d68a Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 18 Jul 2024 11:02:23 +0800 Subject: [PATCH 02/11] tests: add Test Management Tool kdump NFS tests Currently, all integration tests fail F40 because of failing to install package and enable services on the cloud image properly. TMT (Test Management Tool) supports running tests on virtual machines and provides an user-friendly interface (YAML) to define tasks like installing packages, running commands on the virtual machines. By using TMT, we no longer need to spend time and efforts on maintaining the current test framework and can also enjoy rich handy features like managing tests provided by TMT. There are six test steps in TMT. Quoting [1], here is an overview, - discover Gather information about test cases to be executed. - provision Provision an environment for testing or use localhost. - prepare Prepare the environment for testing. - execute Run tests using the specified executor. - report Provide test results overview and send reports. - finish Perform the finishing tasks and clean up provisioned guests. And each step step is configurable via the so-called plans [2]. TMT also uses plans to group relevant tests and select needed tests. This commit adds the test of sending crash dump to a NFS target by setting two VMs using TMT's MultiHost Testing feature [3], - client Setup up NFS dump target and trigger crash - server acts as an NFS server and check sent vmcore The scaffold of TMT test project can be created as follows, mkdir tests && cd tests tmt init --template mini mv plans/example/{example,kdump}.fmf tmt tests create /setup/default_crashkernel -t beakerlib tmt tests create /setup/nfs_server -t beakerlib tmt tests create /test/trigger_crash -t beakerlib tmt tests create /test/check_vmcore -t beakerlib Then we fill in the content for created plans and tests, # tmt run -a /var/tmp/tmt/run-017 /plans/kdump discover how: fmf name: client-setup directory: /root/kdump-tests tests: /setup/kdump how: fmf name: server-setup ... execute task #4: server-test on server how: tmt summary: 4 tests executed report how: display summary: 4 tests passed finish A main.fmf is put under the root directory of tests and setup for other tests to inherit [4]. When there is a kernel panic, ssh access to the guest will exit with with 255 and TMT will retreat it as an error. We can use restart-on-exit-code=255 [5] to let TMT this is expected. [1] https://tmt.readthedocs.io/en/stable/overview.html#run [2] https://tmt.readthedocs.io/en/stable/spec/plans.html [3] https://tmt.readthedocs.io/en/stable/guide.html#multihost-testing [4] https://tmt.readthedocs.io/en/stable/guide.html#inheritance [5] https://tmt.readthedocs.io/en/latest/spec/tests.html#restart Signed-off-by: Coiby Xu --- tests/.editorconfig | 9 ++++ tests/.fmf/version | 1 + tests/plans/nfs.fmf | 39 +++++++++++++++ tests/setup/default_crashkernel/main.fmf | 5 ++ tests/setup/default_crashkernel/test.sh | 22 +++++++++ tests/setup/main.fmf | 2 + tests/setup/nfs_server/main.fmf | 3 ++ tests/setup/nfs_server/test.sh | 15 ++++++ tests/setup/trigger_crash/main.fmf | 3 ++ tests/setup/trigger_crash/test.sh | 62 ++++++++++++++++++++++++ tests/tests/check_vmcore/main.fmf | 2 + tests/tests/check_vmcore/test.sh | 56 +++++++++++++++++++++ tests/tests/main.fmf | 2 + 13 files changed, 221 insertions(+) create mode 100644 tests/.editorconfig create mode 100644 tests/.fmf/version create mode 100644 tests/plans/nfs.fmf create mode 100644 tests/setup/default_crashkernel/main.fmf create mode 100755 tests/setup/default_crashkernel/test.sh create mode 100644 tests/setup/main.fmf create mode 100644 tests/setup/nfs_server/main.fmf create mode 100755 tests/setup/nfs_server/test.sh create mode 100644 tests/setup/trigger_crash/main.fmf create mode 100755 tests/setup/trigger_crash/test.sh create mode 100644 tests/tests/check_vmcore/main.fmf create mode 100755 tests/tests/check_vmcore/test.sh create mode 100644 tests/tests/main.fmf diff --git a/tests/.editorconfig b/tests/.editorconfig new file mode 100644 index 00000000..ca46508f --- /dev/null +++ b/tests/.editorconfig @@ -0,0 +1,9 @@ +# Use dracut code style +[*] +shell_variant = bash +indent_style = space +indent_size = 4 +switch_case_indent = true +function_next_line = false +binary_next_line = true +space_redirects = true diff --git a/tests/.fmf/version b/tests/.fmf/version new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/tests/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf new file mode 100644 index 00000000..69f1e29f --- /dev/null +++ b/tests/plans/nfs.fmf @@ -0,0 +1,39 @@ +summary: Kdump NFS dumping tests +discover: + - name: Set up crashkernel + how: fmf + test: + - setup/default_crashkernel + where: + - client + - name: Set up NFS server + how: fmf + test: + - setup/nfs_server + where: + - server + - name: Panic kernel + how: fmf + test: + - /setup/trigger_crash + where: + - client + - name: Check vmcore + how: fmf + test: + - /tests/check_vmcore + where: + - server + +provision: + - name: server + how: virtual + connection: system + + - name: client + how: virtual + connection: system + +execute: + how: tmt + exit-first: true diff --git a/tests/setup/default_crashkernel/main.fmf b/tests/setup/default_crashkernel/main.fmf new file mode 100644 index 00000000..675a1ace --- /dev/null +++ b/tests/setup/default_crashkernel/main.fmf @@ -0,0 +1,5 @@ +summary: Install kdump-utils and set up the crashkernel kernel parameter +require: +- kdump-utils +# Install the squashfs kernel module for the default crashkernel value to work +- kernel-modules-core diff --git a/tests/setup/default_crashkernel/test.sh b/tests/setup/default_crashkernel/test.sh new file mode 100755 index 00000000..42cb3976 --- /dev/null +++ b/tests/setup/default_crashkernel/test.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k + +# shellcheck source=/dev/null +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +rlJournalStart + +if [ "$TMT_REBOOT_COUNT" == 0 ]; then + rlPhaseStartTest + rlRun "kdumpctl reset-crashkernel --kernel=ALL" + rlRun "tmt-reboot" + rlPhaseEnd + +elif [ "$TMT_REBOOT_COUNT" == 1 ]; then + rlPhaseStartTest + _default_crashkernel=$(kdumpctl get-default-crashkernel) + rlRun "grep crashkernel=$_default_crashkernel /proc/cmdline" + rlPhaseEnd +fi + +rlJournalEnd diff --git a/tests/setup/main.fmf b/tests/setup/main.fmf new file mode 100644 index 00000000..3fede6bd --- /dev/null +++ b/tests/setup/main.fmf @@ -0,0 +1,2 @@ +test: ./test.sh +framework: beakerlib diff --git a/tests/setup/nfs_server/main.fmf b/tests/setup/nfs_server/main.fmf new file mode 100644 index 00000000..9d68577d --- /dev/null +++ b/tests/setup/nfs_server/main.fmf @@ -0,0 +1,3 @@ +summary: Setup NFS server for saving vmcore +require: +- nfs-utils diff --git a/tests/setup/nfs_server/test.sh b/tests/setup/nfs_server/test.sh new file mode 100755 index 00000000..32fb1643 --- /dev/null +++ b/tests/setup/nfs_server/test.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k + +# shellcheck source=/dev/null +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +rlJournalStart +rlPhaseStartTest +rlRun "dnf -y install nfs-utils" +rlRun "mkdir -p /var/tmp/nfsshare/var/crash" +rlRun "echo '/var/tmp/nfsshare 192.168.0.0/16(rw,no_root_squash)' >> /etc/exports" +rlRun "systemctl enable --now rpcbind nfs-server" +rlPhaseEnd + +rlJournalEnd diff --git a/tests/setup/trigger_crash/main.fmf b/tests/setup/trigger_crash/main.fmf new file mode 100644 index 00000000..8839cf71 --- /dev/null +++ b/tests/setup/trigger_crash/main.fmf @@ -0,0 +1,3 @@ +summary: Dump kernel crash to an NFS server +require: nfs-utils +restart-on-exit-code: 255 diff --git a/tests/setup/trigger_crash/test.sh b/tests/setup/trigger_crash/test.sh new file mode 100755 index 00000000..347ed362 --- /dev/null +++ b/tests/setup/trigger_crash/test.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k + +# shellcheck source=/dev/null +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +# get_IP and assign_server_roles are based on keylime-tests +# https://github.com/RedHat-SP-Security/keylime-tests/blob/e3117dd17bc01c5bf1fcbd8986d0683a0952d738/Multihost/basic-attestation/test.sh#L46 +get_IP() { + if echo "$1" | grep -E -q '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'; then + echo "$1" + else + host "$1" | sed -n -e 's/.*has address //p' | head -n 1 + fi +} + +assign_server_roles() { + if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then + # assign roles based on tmt topology data + cat "${TMT_TOPOLOGY_BASH}" + # shellcheck source=/dev/null + . "${TMT_TOPOLOGY_BASH}" + + CLIENT=${TMT_GUESTS["client.hostname"]} + export CLIENT + SERVER=${TMT_GUESTS["server.hostname"]} + export SERVER + MY_IP="${TMT_GUEST['hostname']}" + elif [ -n "$SERVERS" ]; then + # assign roles using SERVERS and CLIENTS variables + # shellcheck disable=SC2153 # CLIENTS is an env variable + SERVER=$(echo "$SERVERS $CLIENTS" | awk '{ print $1 }') + export SERVER + CLIENT=$(echo "$SERVERS $CLIENTS" | awk '{ print $2 }') + export CLIENT + fi + + [ -z "$MY_IP" ] && MY_IP=$(hostname -I | awk '{ print $1 }') + [ -n "$SERVER" ] && SERVER_IP=$(get_IP "$SERVER") && export SERVER_IP + [ -n "$CLIENT" ] && CLIENT_IP=$(get_IP "$CLIENT") && export CLIENT_IP +} + +rlJournalStart + +if [ "$TMT_TEST_RESTART_COUNT" == 0 ]; then + rlPhaseStartSetup + assign_server_roles + rlLog "SERVER: $SERVER ${SERVER_IP}" + rlLog "CLIENT: ${CLIENT} ${CLIENT_IP}" + rlLog "This system is: $(hostname) ${MY_IP}" + rlRun "rm -rf /var/tmp/nfsshare/var/crash/*" + rlRun "echo nfs ${SERVER}:/var/tmp/nfsshare > /etc/kdump.conf" + rlPhaseEnd + + rlPhaseStartTest + rlRun "kdumpctl restart" || rlDie "Failed to restart kdump" + rlRun "sync" + rlRun "echo 1 > /proc/sys/kernel/sysrq" + rlRun "echo c > /proc/sysrq-trigger" + rlPhaseEnd +fi +rlJournalEnd diff --git a/tests/tests/check_vmcore/main.fmf b/tests/tests/check_vmcore/main.fmf new file mode 100644 index 00000000..a51e23ca --- /dev/null +++ b/tests/tests/check_vmcore/main.fmf @@ -0,0 +1,2 @@ +summary: Check vmcore on NFS server +require: makedumpfile diff --git a/tests/tests/check_vmcore/test.sh b/tests/tests/check_vmcore/test.sh new file mode 100755 index 00000000..8d56269c --- /dev/null +++ b/tests/tests/check_vmcore/test.sh @@ -0,0 +1,56 @@ +#!/bin/bash -eux + +has_valid_vmcore_dir() { + local path=$1 + local vmcore_dir + local vmcore="" + + MAX_WAIT_TIME=300 + _waited=0 + while [ $_waited -le $MAX_WAIT_TIME ]; do + if vmcore_dir=$(find "$path/"* -maxdepth 0 2> /dev/null); then + break + fi + sleep 1 + _waited=$((_waited + 1)) + done + + if [[ $_waited -ge $MAX_WAIT_TIME ]]; then + echo "No Vmcore found!" 1>&2 + return 1 + fi + + echo "Found a vmcore dir \"$vmcore_dir\":" + # Checking with `crash` is slow and consume a lot of memory/disk, + # just do a sanity check by check if log are available. + if [ -e "$vmcore_dir/vmcore" ]; then + vmcore=$vmcore_dir/vmcore + makedumpfile --dump-dmesg "$vmcore" "$vmcore_dir/vmcore-dmesg.txt.2" || { + echo "Failed to retrive dmesg from vmcore!" + return 1 + } + elif [ -e "$vmcore_dir/vmcore.flat" ]; then + vmcore=$vmcore_dir/vmcore.flat + makedumpfile -R "$vmcore_dir/vmcore" < "$vmcore" || return 1 + makedumpfile --dump-dmesg "$vmcore_dir/vmcore" "$vmcore_dir/vmcore-dmesg.txt.2" || { + echo "Failed to retrive dmesg from vmcore!" + return 1 + } + rm "$vmcore_dir/vmcore" + else + echo "The vmcore dir is empty!" + return 1 + fi + + if ! diff -w $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then + echo "Dmesg retrived from vmcore is different from dump version!" + return 1 + fi + + echo "VMCORE: $vmcore" + echo "KERNEL VERSION: $(rpm -q kernel-core)" + + return 0 +} + +has_valid_vmcore_dir "/var/tmp/nfsshare/var/crash/" diff --git a/tests/tests/main.fmf b/tests/tests/main.fmf new file mode 100644 index 00000000..b5df9c66 --- /dev/null +++ b/tests/tests/main.fmf @@ -0,0 +1,2 @@ +test: ./test.sh +framework: shell From 52f33a5d1d25cd0a645b3287614b465160576f9f Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Wed, 31 Jul 2024 05:07:51 +0800 Subject: [PATCH 03/11] tests: Don't compare vmcore-dmesg.txt for now Somehow the dmesg log extracted by makedumpfile vmcore-dmesg.2.txt has "[ T[0-9]]" like follows, --- /var/crash/192.168.122.27-2024-07-30-21:01:49/vmcore-dmesg.txt 2024-07-30 21:01:51.865079103 +0000 +++ /var/crash/192.168.122.27-2024-07-30-21:01:49/vmcore-dmesg.txt.2 2024-07-30 21:02:05.504060793 +0000 @@ -1,830 +1,830 @@ -[ 0.000000] Linux version 6.8.5-301.fc40.x86_64 (mockbuild@0bc0cc78c12e4762acf61c209bd02e96) (gcc (GCC) 14.0.1 20240328 (Red Hat 14.0.1-0), GNU ld version 2.41-34.fc40) #1 SMP PREEMPT_DYNAMIC Thu Apr 11 20:00:10 UTC 2024 -[ 0.000000] Command line: BOOT_IMAGE=(hd0,gpt3)/vmlinuz-6.8.5-301.fc40.x86_64 no_timer_check net.ifnames=0 console=tty1 console=ttyS0,115200n8 root=UUID=8424dfd0-f878-4302-8ee5-b2ec6e5eb868 rootflags=subvol=root crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M -[ 0.000000] BIOS-provided physical RAM map: -[ 0.000000] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable -[ 0.000000] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved -[ 0.000000] BIOS-e820: [mem 0x00000000000f0000-0x00000000000fffff] reserved ... +[ 0.000000] [ T0] Linux version 6.8.5-301.fc40.x86_64 (mockbuild@0bc0cc78c12e4762acf61c209bd02e96) (gcc (GCC) 14.0.1 20240328 (Red Hat 14.0.1-0), GNU ld version 2.41-34.fc40) #1 SMP PREEMPT_DYNAMIC Thu Apr 11 20:00:10 UTC 2024 +[ 0.000000] [ T0] Command line: BOOT_IMAGE=(hd0,gpt3)/vmlinuz-6.8.5-301.fc40.x86_64 no_timer_check net.ifnames=0 console=tty1 console=ttyS0,115200n8 root=UUID=8424dfd0-f878-4302-8ee5-b2ec6e5eb868 rootflags=subvol=root crashkernel=1G-4G:192M,4G-64G:256M,64G-:512M +[ 0.000000] [ T0] BIOS-provided physical RAM map: +[ 0.000000] [ T0] BIOS-e820: [mem 0x0000000000000000-0x000000000009fbff] usable +[ 0.000000] [ T0] BIOS-e820: [mem 0x000000000009fc00-0x000000000009ffff] reserved +[ 0.000000] [ T0] BIOS-e820: [mem 0x00000000000f0000-0x00000000000fffff] reserved Signed-off-by: Coiby Xu --- tests/tests/check_vmcore/test.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/tests/check_vmcore/test.sh b/tests/tests/check_vmcore/test.sh index 8d56269c..15d3d2ce 100755 --- a/tests/tests/check_vmcore/test.sh +++ b/tests/tests/check_vmcore/test.sh @@ -42,10 +42,11 @@ has_valid_vmcore_dir() { return 1 fi - if ! diff -w $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then - echo "Dmesg retrived from vmcore is different from dump version!" - return 1 - fi + # Don't compare vmcore-dmesg.txt for now as "[ T[0-9]" somehow appears in vmcore-dmesg.txt.2 + # if ! diff -w $vmcore_dir/vmcore-dmesg.txt.2 $vmcore_dir/vmcore-dmesg.txt; then + # echo "Dmesg retrived from vmcore is different from dump version!" + # return 1 + # fi echo "VMCORE: $vmcore" echo "KERNEL VERSION: $(rpm -q kernel-core)" From fe06faa6b834f053b97ee3774cca3522d206d045 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Thu, 18 Jul 2024 11:33:26 +0800 Subject: [PATCH 04/11] tests: set root password to make it easier to troubleshot test failures To make it easier to troubleshot test failures, set root password as kdump. Then we can use "virsh console VM" to have console access to a VM. Note we can also have ssh access to the VMs using tmt [1]. For example, "tmt run login --step prepare" allows you to access the VM at the prepare step. [1] https://tmt.readthedocs.io/en/stable/examples.html#guest-login Signed-off-by: Coiby Xu --- tests/plans/nfs.fmf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf index 69f1e29f..bb47c469 100644 --- a/tests/plans/nfs.fmf +++ b/tests/plans/nfs.fmf @@ -34,6 +34,13 @@ provision: how: virtual connection: system +prepare: + # Set root password to log in as root in the console + - name: Set root password + how: shell + script: + - echo root:kdump | chpasswd + execute: how: tmt exit-first: true From ee31867be24fdef3d4687e8a3bb8ccfbf473e1e1 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Wed, 24 Jul 2024 18:11:12 +0800 Subject: [PATCH 05/11] tests: allow to use custom mirror Currently it takes about half an hour to finish the test on dell-per340-03. If we use a faster mirror, e.g. https://mirrors.tuna.tsinghua.edu.cn/fedora, only 4m is needed. Support using custom mirror by "tmt run --environment" e.g. tmt run --environment CUSTOM_MIRROR=https://mirrors.tuna.tsinghua.edu.cn/fedora ... Note fedora-cisco-openh264 is disabled because it's not needed. Signed-off-by: Coiby Xu --- tests/plans/nfs.fmf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf index bb47c469..6e4b354a 100644 --- a/tests/plans/nfs.fmf +++ b/tests/plans/nfs.fmf @@ -41,6 +41,12 @@ prepare: script: - echo root:kdump | chpasswd + - name: Use custom mirror + how: shell + script: + - test -v CUSTOM_MIRROR && sed -e 's/^metalink=/#metalink=/g' -e "s|^#baseurl=http://download.example/pub/fedora/linux|baseurl=${CUSTOM_MIRROR}|g" -i.bak /etc/yum.repos.d/fedora*.repo || true + - dnf config-manager --set-disabled fedora-cisco-openh264 || true + execute: how: tmt exit-first: true From 44329e5b8932da0af2a8bb7757f1bc565633f992 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Mon, 22 Jul 2024 18:07:10 +0800 Subject: [PATCH 06/11] tests: install a custom built kdump-utils RPM We can use "tmt run --environment KDUMP_UTILS_RPM=PATH_TO_RPM" to install custom built kdump-utils. Signed-off-by: Coiby Xu --- tests/plans/nfs.fmf | 7 +++++++ tests/setup/default_crashkernel/main.fmf | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf index 6e4b354a..3b944469 100644 --- a/tests/plans/nfs.fmf +++ b/tests/plans/nfs.fmf @@ -47,6 +47,13 @@ prepare: - test -v CUSTOM_MIRROR && sed -e 's/^metalink=/#metalink=/g' -e "s|^#baseurl=http://download.example/pub/fedora/linux|baseurl=${CUSTOM_MIRROR}|g" -i.bak /etc/yum.repos.d/fedora*.repo || true - dnf config-manager --set-disabled fedora-cisco-openh264 || true + - name: Install built RPM + how: install + package: + - "$KDUMP_UTILS_RPM" + where: + - client + execute: how: tmt exit-first: true diff --git a/tests/setup/default_crashkernel/main.fmf b/tests/setup/default_crashkernel/main.fmf index 675a1ace..eb3b4c2d 100644 --- a/tests/setup/default_crashkernel/main.fmf +++ b/tests/setup/default_crashkernel/main.fmf @@ -1,5 +1,4 @@ -summary: Install kdump-utils and set up the crashkernel kernel parameter +summary: Set up the crashkernel kernel parameter require: -- kdump-utils # Install the squashfs kernel module for the default crashkernel value to work - kernel-modules-core From 2eb9edcc40646eaa3f30612118e2b9ed54e1d344 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Fri, 26 Jul 2024 16:09:58 +0800 Subject: [PATCH 07/11] tests: add local dumping test plan For local vmcore dumping, we only need to trigger a kernel crash and check if vmcore exists and it can re-use some setups and tests from NFS dumping. Note 1. plans/main.fmf is created for NFS and local dumping test plans to inherit. 2. Virtual tests [1] are created so local dumping can re-use the code of checking vmcore on an NFS server 3. The local test is disabled for now due to [2] 4. exit-first=true [3] is to make the executor stop executing tests once a test failure is encountered. But TMT currently has the problem of exit-first not applying across discover phase boundaries [4]. So for local dumping test plan, simply remove multiple discover phases [5]. [1] https://tmt.readthedocs.io/en/stable/guide.html#virtual-tests [2] https://bugzilla.redhat.com/show_bug.cgi?id=2270423 [3] https://tmt.readthedocs.io/en/stable/spec/plans.html#exit-first [4] https://github.com/teemtee/tmt/issues/3116 [5] https://tmt.readthedocs.io/en/stable/guide.html#multihost-testing Signed-off-by: Coiby Xu --- tests/plans/local.fmf | 14 ++++++++ tests/plans/main.fmf | 34 ++++++++++++++++++ tests/plans/nfs.fmf | 41 ++++------------------ tests/setup/nfs_client/main.fmf | 2 ++ tests/setup/nfs_client/test.sh | 56 ++++++++++++++++++++++++++++++ tests/setup/trigger_crash/main.fmf | 3 +- tests/setup/trigger_crash/test.sh | 46 +----------------------- tests/tests/check_vmcore/main.fmf | 9 ++++- tests/tests/check_vmcore/test.sh | 4 ++- 9 files changed, 126 insertions(+), 83 deletions(-) create mode 100644 tests/plans/local.fmf create mode 100644 tests/plans/main.fmf create mode 100644 tests/setup/nfs_client/main.fmf create mode 100755 tests/setup/nfs_client/test.sh diff --git a/tests/plans/local.fmf b/tests/plans/local.fmf new file mode 100644 index 00000000..f193197b --- /dev/null +++ b/tests/plans/local.fmf @@ -0,0 +1,14 @@ +summary: Kdump local dumping +# Disable this plan due to https://bugzilla.redhat.com/show_bug.cgi?id=2270423 +enabled: false +discover: + how: fmf + test: + - /setup/default_crashkernel + - /setup/trigger_crash + - /tests/check_vmcore/local + +provision: + - name: client + how: virtual + connection: system diff --git a/tests/plans/main.fmf b/tests/plans/main.fmf new file mode 100644 index 00000000..16dbd833 --- /dev/null +++ b/tests/plans/main.fmf @@ -0,0 +1,34 @@ +provision: + - name: server + how: virtual + connection: system + + - name: client + how: virtual + connection: system + + +prepare: + # Set root password to log in as root in the console + - name: Set root password + how: shell + script: + - echo root:kdump | chpasswd + + - name: Use custom mirror + how: shell + script: + - test -v CUSTOM_MIRROR && sed -e 's/^metalink=/#metalink=/g' -e "s|^#baseurl=http://download.example/pub/fedora/linux|baseurl=${CUSTOM_MIRROR}|g" -i.bak /etc/yum.repos.d/fedora*.repo || true + - dnf config-manager --set-disabled fedora-cisco-openh264 || true + + - name: Install built RPM + how: install + package: + - "$KDUMP_UTILS_RPM" + where: + - client + + +execute: + how: tmt + exit-first: true diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf index 3b944469..35e18c99 100644 --- a/tests/plans/nfs.fmf +++ b/tests/plans/nfs.fmf @@ -12,6 +12,12 @@ discover: - setup/nfs_server where: - server + - name: Setup NFS client + how: fmf + test: + - setup/nfs_client + where: + - client - name: Panic kernel how: fmf test: @@ -21,39 +27,6 @@ discover: - name: Check vmcore how: fmf test: - - /tests/check_vmcore + - /tests/check_vmcore/nfs where: - server - -provision: - - name: server - how: virtual - connection: system - - - name: client - how: virtual - connection: system - -prepare: - # Set root password to log in as root in the console - - name: Set root password - how: shell - script: - - echo root:kdump | chpasswd - - - name: Use custom mirror - how: shell - script: - - test -v CUSTOM_MIRROR && sed -e 's/^metalink=/#metalink=/g' -e "s|^#baseurl=http://download.example/pub/fedora/linux|baseurl=${CUSTOM_MIRROR}|g" -i.bak /etc/yum.repos.d/fedora*.repo || true - - dnf config-manager --set-disabled fedora-cisco-openh264 || true - - - name: Install built RPM - how: install - package: - - "$KDUMP_UTILS_RPM" - where: - - client - -execute: - how: tmt - exit-first: true diff --git a/tests/setup/nfs_client/main.fmf b/tests/setup/nfs_client/main.fmf new file mode 100644 index 00000000..711d2e89 --- /dev/null +++ b/tests/setup/nfs_client/main.fmf @@ -0,0 +1,2 @@ +summary: Set up NFS dumping target +require: nfs-utils diff --git a/tests/setup/nfs_client/test.sh b/tests/setup/nfs_client/test.sh new file mode 100755 index 00000000..c807c13f --- /dev/null +++ b/tests/setup/nfs_client/test.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k + +# shellcheck source=/dev/null +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +# get_IP and assign_server_roles are adapated from keylime-tests +# https://github.com/RedHat-SP-Security/keylime-tests/blob/e3117dd17bc01c5bf1fcbd8986d0683a0952d738/Multihost/basic-attestation/test.sh#L46 +get_IP() { + if echo "$1" | grep -E -q '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'; then + echo "$1" + else + host "$1" | sed -n -e 's/.*has address //p' | head -n 1 + fi +} + +assign_server_roles() { + if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then + # assign roles based on tmt topology data + cat "${TMT_TOPOLOGY_BASH}" + # shellcheck source=/dev/null + . "${TMT_TOPOLOGY_BASH}" + + CLIENT=${TMT_GUESTS["client.hostname"]} + export CLIENT + SERVER=${TMT_GUESTS["server.hostname"]} + export SERVER + MY_IP="${TMT_GUEST['hostname']}" + elif [ -n "$SERVERS" ]; then + # assign roles using SERVERS and CLIENTS variables + # shellcheck disable=SC2153 # CLIENTS is an env variable + SERVER=$(echo "$SERVERS $CLIENTS" | awk '{ print $1 }') + export SERVER + CLIENT=$(echo "$SERVERS $CLIENTS" | awk '{ print $2 }') + export CLIENT + fi + + [ -z "$MY_IP" ] && MY_IP=$(hostname -I | awk '{ print $1 }') + [ -n "$SERVER" ] && SERVER_IP=$(get_IP "$SERVER") && export SERVER_IP + [ -n "$CLIENT" ] && CLIENT_IP=$(get_IP "$CLIENT") && export CLIENT_IP +} + +rlJournalStart + +rlPhaseStartSetup +assign_server_roles +rlLog "SERVER: $SERVER ${SERVER_IP}" +rlLog "CLIENT: ${CLIENT} ${CLIENT}" +rlLog "This system is: $(hostname) ${MY_IP}" +rlPhaseEnd + +rlPhaseStartTest +rlRun "echo nfs $SERVER:/var/tmp/nfsshare > /etc/kdump.conf" +rlPhaseEnd + +rlJournalEnd diff --git a/tests/setup/trigger_crash/main.fmf b/tests/setup/trigger_crash/main.fmf index 8839cf71..f3bed74d 100644 --- a/tests/setup/trigger_crash/main.fmf +++ b/tests/setup/trigger_crash/main.fmf @@ -1,3 +1,2 @@ -summary: Dump kernel crash to an NFS server -require: nfs-utils +summary: Trigger a kernel panic restart-on-exit-code: 255 diff --git a/tests/setup/trigger_crash/test.sh b/tests/setup/trigger_crash/test.sh index 347ed362..9b86bd5a 100755 --- a/tests/setup/trigger_crash/test.sh +++ b/tests/setup/trigger_crash/test.sh @@ -4,54 +4,9 @@ # shellcheck source=/dev/null . /usr/share/beakerlib/beakerlib.sh || exit 1 -# get_IP and assign_server_roles are based on keylime-tests -# https://github.com/RedHat-SP-Security/keylime-tests/blob/e3117dd17bc01c5bf1fcbd8986d0683a0952d738/Multihost/basic-attestation/test.sh#L46 -get_IP() { - if echo "$1" | grep -E -q '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+'; then - echo "$1" - else - host "$1" | sed -n -e 's/.*has address //p' | head -n 1 - fi -} - -assign_server_roles() { - if [ -n "${TMT_TOPOLOGY_BASH}" ] && [ -f "${TMT_TOPOLOGY_BASH}" ]; then - # assign roles based on tmt topology data - cat "${TMT_TOPOLOGY_BASH}" - # shellcheck source=/dev/null - . "${TMT_TOPOLOGY_BASH}" - - CLIENT=${TMT_GUESTS["client.hostname"]} - export CLIENT - SERVER=${TMT_GUESTS["server.hostname"]} - export SERVER - MY_IP="${TMT_GUEST['hostname']}" - elif [ -n "$SERVERS" ]; then - # assign roles using SERVERS and CLIENTS variables - # shellcheck disable=SC2153 # CLIENTS is an env variable - SERVER=$(echo "$SERVERS $CLIENTS" | awk '{ print $1 }') - export SERVER - CLIENT=$(echo "$SERVERS $CLIENTS" | awk '{ print $2 }') - export CLIENT - fi - - [ -z "$MY_IP" ] && MY_IP=$(hostname -I | awk '{ print $1 }') - [ -n "$SERVER" ] && SERVER_IP=$(get_IP "$SERVER") && export SERVER_IP - [ -n "$CLIENT" ] && CLIENT_IP=$(get_IP "$CLIENT") && export CLIENT_IP -} - rlJournalStart if [ "$TMT_TEST_RESTART_COUNT" == 0 ]; then - rlPhaseStartSetup - assign_server_roles - rlLog "SERVER: $SERVER ${SERVER_IP}" - rlLog "CLIENT: ${CLIENT} ${CLIENT_IP}" - rlLog "This system is: $(hostname) ${MY_IP}" - rlRun "rm -rf /var/tmp/nfsshare/var/crash/*" - rlRun "echo nfs ${SERVER}:/var/tmp/nfsshare > /etc/kdump.conf" - rlPhaseEnd - rlPhaseStartTest rlRun "kdumpctl restart" || rlDie "Failed to restart kdump" rlRun "sync" @@ -59,4 +14,5 @@ if [ "$TMT_TEST_RESTART_COUNT" == 0 ]; then rlRun "echo c > /proc/sysrq-trigger" rlPhaseEnd fi + rlJournalEnd diff --git a/tests/tests/check_vmcore/main.fmf b/tests/tests/check_vmcore/main.fmf index a51e23ca..0fae1b51 100644 --- a/tests/tests/check_vmcore/main.fmf +++ b/tests/tests/check_vmcore/main.fmf @@ -1,2 +1,9 @@ -summary: Check vmcore on NFS server require: makedumpfile + +/nfs: + summary: Check vmcore on NFS server + environment: + VMCORE_PATH: /var/tmp/nfsshare/var/crash + +/local: + summary: Check vmcore locally diff --git a/tests/tests/check_vmcore/test.sh b/tests/tests/check_vmcore/test.sh index 15d3d2ce..6165fa61 100755 --- a/tests/tests/check_vmcore/test.sh +++ b/tests/tests/check_vmcore/test.sh @@ -54,4 +54,6 @@ has_valid_vmcore_dir() { return 0 } -has_valid_vmcore_dir "/var/tmp/nfsshare/var/crash/" +VMCORE_PATH=${VMCORE_PATH:-/var/crash} + +has_valid_vmcore_dir "$VMCORE_PATH" From 803dad409be72b66dc2b7d48d17efc35c9b47336 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Tue, 30 Jul 2024 09:43:33 +0800 Subject: [PATCH 08/11] tests: add SSH dumping test plan TMT stores server SSH key in /var/tmp/tmt/run-*/plans/ssh/provision/server/id_ecdsa. Signed-off-by: Coiby Xu --- tests/plans/nfs.fmf | 2 +- tests/plans/ssh.fmf | 26 +++++++++++++++++++ tests/setup/network_client/main.fmf | 11 ++++++++ .../{nfs_client => network_client}/test.sh | 16 +++++++++++- tests/setup/nfs_client/main.fmf | 2 -- tests/tests/check_vmcore/main.fmf | 3 +++ 6 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 tests/plans/ssh.fmf create mode 100644 tests/setup/network_client/main.fmf rename tests/setup/{nfs_client => network_client}/test.sh (71%) delete mode 100644 tests/setup/nfs_client/main.fmf diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf index 35e18c99..5042f823 100644 --- a/tests/plans/nfs.fmf +++ b/tests/plans/nfs.fmf @@ -15,7 +15,7 @@ discover: - name: Setup NFS client how: fmf test: - - setup/nfs_client + - setup/network_client/nfs where: - client - name: Panic kernel diff --git a/tests/plans/ssh.fmf b/tests/plans/ssh.fmf new file mode 100644 index 00000000..0a3f600d --- /dev/null +++ b/tests/plans/ssh.fmf @@ -0,0 +1,26 @@ +summary: Kdump SSH dumping tests +discover: + - name: Set up crashkernel + how: fmf + test: + - /setup/default_crashkernel + where: + - client + - name: Set up ssh client + how: fmf + test: + - /setup/network_client/ssh + where: + - client + - name: Panic kernel + how: fmf + test: + - /setup/trigger_crash + where: + - client + - name: Check vmcore + how: fmf + test: + - /tests/check_vmcore/ssh + where: + - server diff --git a/tests/setup/network_client/main.fmf b/tests/setup/network_client/main.fmf new file mode 100644 index 00000000..0e7182c1 --- /dev/null +++ b/tests/setup/network_client/main.fmf @@ -0,0 +1,11 @@ + +/nfs: + summary: Set up NFS dumping target + require: nfs-utils + environment: + REMOTE_TYPE: NFS + +/ssh: + summary: Set up SSH dumping target + environment: + REMOTE_TYPE: SSH diff --git a/tests/setup/nfs_client/test.sh b/tests/setup/network_client/test.sh similarity index 71% rename from tests/setup/nfs_client/test.sh rename to tests/setup/network_client/test.sh index c807c13f..436635e7 100755 --- a/tests/setup/nfs_client/test.sh +++ b/tests/setup/network_client/test.sh @@ -50,7 +50,21 @@ rlLog "This system is: $(hostname) ${MY_IP}" rlPhaseEnd rlPhaseStartTest -rlRun "echo nfs $SERVER:/var/tmp/nfsshare > /etc/kdump.conf" +if [[ $REMOTE_TYPE == NFS ]]; then + rlRun "echo nfs $SERVER:/var/tmp/nfsshare > /etc/kdump.conf" +elif [[ $REMOTE_TYPE == SSH ]]; then + TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} + SERVER_SSH_KEY=${TMT_TEST_PLAN_ROOT}/provision/server/id_ecdsa + if test -f "$SERVER_SSH_KEY"; then + rlRun "ssh-keyscan -H $SERVER > /root/.ssh/known_hosts" + rlRun "ssh root@$SERVER -i $SERVER_SSH_KEY 'mkdir /var/crash'" + rlRun "echo ssh root@$SERVER > /etc/kdump.conf" + rlRun "echo sshkey $SERVER_SSH_KEY >> /etc/kdump.conf" + rlRun "echo core_collector makedumpfile -l --message-level 7 -d 31 -F >> /etc/kdump.conf" + else + rlDie "Server SSH Key not found, something wrong" + fi +fi rlPhaseEnd rlJournalEnd diff --git a/tests/setup/nfs_client/main.fmf b/tests/setup/nfs_client/main.fmf deleted file mode 100644 index 711d2e89..00000000 --- a/tests/setup/nfs_client/main.fmf +++ /dev/null @@ -1,2 +0,0 @@ -summary: Set up NFS dumping target -require: nfs-utils diff --git a/tests/tests/check_vmcore/main.fmf b/tests/tests/check_vmcore/main.fmf index 0fae1b51..dda7181b 100644 --- a/tests/tests/check_vmcore/main.fmf +++ b/tests/tests/check_vmcore/main.fmf @@ -7,3 +7,6 @@ require: makedumpfile /local: summary: Check vmcore locally + +/ssh: + summary: Check vmcore on SSH SERVER From 77f3c850ae6dfa25e2608052e596c847b842d38a Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Tue, 30 Jul 2024 15:25:31 +0800 Subject: [PATCH 09/11] tests: add NFS early dumping test plan To avoid triggering the kernel panic repeatedly, this test use kexec reboot to only boot into the initrd with early kdump enabled once. Signed-off-by: Coiby Xu --- tests/plans/nfs.fmf | 3 ++- tests/plans/nfs_early.fmf | 26 ++++++++++++++++++++++++++ tests/setup/network_client/main.fmf | 7 +++++++ tests/setup/network_client/test.sh | 20 ++++++++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/plans/nfs_early.fmf diff --git a/tests/plans/nfs.fmf b/tests/plans/nfs.fmf index 5042f823..8b95c445 100644 --- a/tests/plans/nfs.fmf +++ b/tests/plans/nfs.fmf @@ -15,7 +15,8 @@ discover: - name: Setup NFS client how: fmf test: - - setup/network_client/nfs + # For tmt, nfs means nfs*, so use nfs$ to exclude nfs_early + - setup/network_client/nfs$ where: - client - name: Panic kernel diff --git a/tests/plans/nfs_early.fmf b/tests/plans/nfs_early.fmf new file mode 100644 index 00000000..bd2f294d --- /dev/null +++ b/tests/plans/nfs_early.fmf @@ -0,0 +1,26 @@ +summary: Kdump NFS early dumping tests +discover: + - name: Set up crashkernel + how: fmf + test: + - setup/default_crashkernel + where: + - client + - name: Set up NFS server + how: fmf + test: + - setup/nfs_server + where: + - server + - name: Set up NFS client + how: fmf + test: + - setup/network_client/nfs_early + where: + - client + - name: Check vmcore + how: fmf + test: + - /tests/check_vmcore/nfs + where: + - server diff --git a/tests/setup/network_client/main.fmf b/tests/setup/network_client/main.fmf index 0e7182c1..b73d4146 100644 --- a/tests/setup/network_client/main.fmf +++ b/tests/setup/network_client/main.fmf @@ -5,6 +5,13 @@ environment: REMOTE_TYPE: NFS +/nfs_early: + summary: Set up NFS early dumping target + require: + - nfs-utils + environment: + REMOTE_TYPE: NFS_EARLY + /ssh: summary: Set up SSH dumping target environment: diff --git a/tests/setup/network_client/test.sh b/tests/setup/network_client/test.sh index 436635e7..ca66950a 100755 --- a/tests/setup/network_client/test.sh +++ b/tests/setup/network_client/test.sh @@ -52,6 +52,26 @@ rlPhaseEnd rlPhaseStartTest if [[ $REMOTE_TYPE == NFS ]]; then rlRun "echo nfs $SERVER:/var/tmp/nfsshare > /etc/kdump.conf" +elif [[ $REMOTE_TYPE == NFS_EARLY ]]; then + if [ "$TMT_REBOOT_COUNT" == 0 ]; then + echo "nfs $SERVER:/var/tmp/nfsshare" > /etc/kdump.conf + echo core_collector makedumpfile -l --message-level 7 -d 31 >> /etc/kdump.conf + kdumpctl start || exit 1 + earlykdump_path="/usr/lib/dracut/modules.d/99earlykdump/early-kdump.sh" + tmp_file="/tmp/.tmp-file" + cat << EOF > $tmp_file +echo 1 > /proc/sys/kernel/sysrq +echo c > /proc/sysrq-trigger +EOF + sed -i "/early_kdump_load$/r $tmp_file" $earlykdump_path + cp "/boot/initramfs-$(uname -r).img"{,.bak} + dracut -f --add earlykdump + mv "/boot/initramfs-$(uname -r).img"{,.new} + mv "/boot/initramfs-$(uname -r).img"{.bak,} + sync + kexec -s -l "/boot/vmlinuz-$(uname -r)" --initrd="/boot/initramfs-$(uname -r).img.new" --reuse-cmdline --append=rd.earlykdump + tmt-reboot -c "systemctl kexec" + fi elif [[ $REMOTE_TYPE == SSH ]]; then TMT_TEST_PLAN_ROOT=${TMT_PLAN_DATA%data} SERVER_SSH_KEY=${TMT_TEST_PLAN_ROOT}/provision/server/id_ecdsa From 010be0f6e9f70bac68bdea507604f1fd1931ee8e Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Tue, 30 Jul 2024 16:22:29 +0800 Subject: [PATCH 10/11] tests: add LVM2 thin provision dumping test plan Use hardware.disk[1] to specify an additional disk for LVM2 thin provision as follows, provision: - name: client how: virtual connection: system hardware: disk: - size: = 40GB - size: = 1GB Note tmt doesn't support specify a size of smaller unit like 300MB. [1] https://tmt.readthedocs.io/en/stable/spec/hardware.html#disk Signed-off-by: Coiby Xu --- tests/plans/lvm2_thinp.fmf | 17 ++++++++++++ tests/setup/lvm2_thinp/main.fmf | 3 +++ tests/setup/lvm2_thinp/test.sh | 43 +++++++++++++++++++++++++++++++ tests/tests/check_vmcore/main.fmf | 7 +++++ tests/tests/check_vmcore/test.sh | 5 ++++ 5 files changed, 75 insertions(+) create mode 100644 tests/plans/lvm2_thinp.fmf create mode 100644 tests/setup/lvm2_thinp/main.fmf create mode 100755 tests/setup/lvm2_thinp/test.sh diff --git a/tests/plans/lvm2_thinp.fmf b/tests/plans/lvm2_thinp.fmf new file mode 100644 index 00000000..75934886 --- /dev/null +++ b/tests/plans/lvm2_thinp.fmf @@ -0,0 +1,17 @@ +summary: Kdump LVM2 provision dumping +discover: + how: fmf + test: + - /setup/default_crashkernel + - /setup/lvm2_thinp + - /setup/trigger_crash + - /tests/check_vmcore/lvm2_thinp + +provision: + - name: client + how: virtual + connection: system + hardware: + disk: + - size: = 40GB + - size: = 1GB diff --git a/tests/setup/lvm2_thinp/main.fmf b/tests/setup/lvm2_thinp/main.fmf new file mode 100644 index 00000000..d9e5df4e --- /dev/null +++ b/tests/setup/lvm2_thinp/main.fmf @@ -0,0 +1,3 @@ +summary: Set up LVM2 thin-provision +require: +- lvm2 diff --git a/tests/setup/lvm2_thinp/test.sh b/tests/setup/lvm2_thinp/test.sh new file mode 100755 index 00000000..53f6c7e2 --- /dev/null +++ b/tests/setup/lvm2_thinp/test.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k + +# shellcheck source=/dev/null +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +rlJournalStart + +if [ "$TMT_REBOOT_COUNT" == 0 ]; then + rlPhaseStartTest + VG=vg00 + LV_THINPOOL=thinpool + LV_VOLUME=thinlv + VMCORE_PATH=/var/crash + + cat << EOF > /etc/lvm/lvm.conf +activation { + thin_pool_autoextend_threshold = 70 + thin_pool_autoextend_percent = 20 + monitoring = 1 +} +EOF + + vgcreate $VG /dev/vdb + # Create a small thinpool which is definitely not enough for + # vmcore, then create a thin volume which is definitely enough + # for vmcore, so we can make sure thinpool should be autoextend + # during runtime. + lvcreate -L 10M -T $VG/$LV_THINPOOL + lvcreate -V 300M -T $VG/$LV_THINPOOL -n $LV_VOLUME + mkfs.ext4 /dev/$VG/$LV_VOLUME + mount /dev/$VG/$LV_VOLUME /mnt + mkdir -p /mnt/$VMCORE_PATH + + cat << EOF > /etc/kdump.conf +ext4 /dev/$VG/$LV_VOLUME +path $VMCORE_PATH +core_collector makedumpfile -l --message-level 7 -d 31 +EOF + rlPhaseEnd +fi + +rlJournalEnd diff --git a/tests/tests/check_vmcore/main.fmf b/tests/tests/check_vmcore/main.fmf index dda7181b..ca7d9fb5 100644 --- a/tests/tests/check_vmcore/main.fmf +++ b/tests/tests/check_vmcore/main.fmf @@ -8,5 +8,12 @@ require: makedumpfile /local: summary: Check vmcore locally +/lvm2_thinp: + summary: Check vmcore saved to LVM2 thin provision target + environment: + IS_LVM_THINP: YES + VMCORE_PATH: /mnt/var/crash + + /ssh: summary: Check vmcore on SSH SERVER diff --git a/tests/tests/check_vmcore/test.sh b/tests/tests/check_vmcore/test.sh index 6165fa61..44b38aa3 100755 --- a/tests/tests/check_vmcore/test.sh +++ b/tests/tests/check_vmcore/test.sh @@ -56,4 +56,9 @@ has_valid_vmcore_dir() { VMCORE_PATH=${VMCORE_PATH:-/var/crash} +IS_LVM_THINP=${IS_LVM_THINP:-NO} +if [[ $IS_LVM_THINP == YES ]]; then + mount /dev/vg00/thinlv /mnt +fi + has_valid_vmcore_dir "$VMCORE_PATH" From 934f62476aebdb94437b901323cabfbeb5688858 Mon Sep 17 00:00:00 2001 From: Coiby Xu Date: Fri, 15 Apr 2022 18:15:17 +0800 Subject: [PATCH 11/11] tests: enable Github action With this patch, four kinds of tests are triggered when there is a pull request created in a Github kdump-uitls repo, - format check using shfmt - static analysis using shellcheck - ShellSpec unit tests (test cases in spec/) - integration tests (tests cases in tests/) Signed-off-by: Coiby Xu --- .github/workflows/main.yml | 54 ++++++++++++++++++++++++++++++++++ tools/run-integration-tests.sh | 30 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100755 tools/run-integration-tests.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..fd7e3c5f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,54 @@ +name: kdump-utils tests + +on: pull_request + +jobs: + format-check: + runs-on: ubuntu-latest + container: docker.io/fedora:latest + steps: + - uses: actions/checkout@v4 + - run: dnf install shfmt -y + - run: shfmt -s -d *.sh kdumpctl mk*dumprd tests/*/*/*.sh dracut/*/*.sh tools/*.sh + + static-analysis: + runs-on: ubuntu-latest + container: docker.io/fedora:latest + steps: + - uses: actions/checkout@v4 + - run: dnf install shellcheck -y + # Currently, for kdump-utils, there is need for shellcheck to require + # the sourced file to give correct warnings about the checked file + - run: shellcheck -x *.sh spec/*.sh kdumpctl mk*dumprd + - run: shellcheck -x dracut/*/*.sh + - run: shellcheck -x tests/*/*/*.sh tools/*.sh + + unit-tests: + runs-on: ubuntu-latest + container: docker.io/fedora:latest + steps: + - uses: actions/checkout@v4 + - run: sudo dnf install -y make dracut grubby hostname + - run: curl -L -O https://github.com/shellspec/shellspec/archive/latest.tar.gz && tar -xzf latest.tar.gz + - run: cd shellspec-latest && sudo make install + - run: shellspec + + integration-tests: + runs-on: self-hosted + timeout-minutes: 45 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.fedora_version }}-${{ matrix.test }} + cancel-in-progress: true + strategy: + matrix: + fedora_version: [ + "40", + ] + fail-fast: false + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: "${{ matrix.fedora_version }} kdump tests" + run: tools/run-integration-tests.sh ${{ matrix.fedora_version }} diff --git a/tools/run-integration-tests.sh b/tools/run-integration-tests.sh new file mode 100755 index 00000000..ffd760ae --- /dev/null +++ b/tools/run-integration-tests.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -ex + +[[ -d ${0%/*} ]] && cd "${0%/*}"/../ + +fedora_version=${1:-40} + +dist_abbr=.fc$fedora_version + +VERSION=$(rpmspec -q --queryformat "%{VERSION}" kdump-utils.spec) +SRC_ARCHIVE=kdump-utils-$VERSION.tar.gz +if ! git archive --format=tar.gz -o "$SRC_ARCHIVE" --prefix="kdump-utils-$VERSION/" HEAD; then + echo "Failed to create kdump-utils source archive" + exit 1 +fi + +if ! rpmbuild -ba -D "dist $dist_abbr" -D "_sourcedir $(pwd)" -D "_builddir $(pwd)" -D "_srcrpmdir $(pwd)" -D "_rpmdir $(pwd)" kdump-utils.spec; then + echo "Failed to build kdump-utils rpm" + exit 1 +fi + +arch=$(uname -m) +rpm_name=$(rpmspec -D "dist $dist_abbr" -q --queryformat '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' kdump-utils.spec) +rpm_path="$(pwd)/${arch}/${rpm_name}.rpm" +if [[ ! -f $rpm_path ]]; then + echo "Failed to find built kdump-utils rpm ($rpm_path doesn't eixst)" +fi + +cd tests && tmt run --environment CUSTOM_MIRROR=https://mirrors.tuna.tsinghua.edu.cn/fedora --environment KDUMP_UTILS_RPM="$rpm_path" -a provision -h virtual -i fedora:"$fedora_version"