From 28d7c34d350df43e4370a99a48d3d59528b681bc Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Mon, 20 May 2024 11:07:36 +0800 Subject: [PATCH] kdumpctl: add kdump tested status updating/reporting support Motivation ========== People usually won't test if kdump can really generate a vmcore before regarding kdump as workable, which as a result, a possibility of no vmcores generated after a real system crash. It is unexpected for kdump. Thought it is highly recommented people to test kdump after any system modification, such as: a. after kernel patching or whole yum update, as it might break something on which kdump is dependent, maybe due to introduction of any new bug etc. b. after any change at hardware level, maybe storage, networking, firmware upgrading etc. c. after implementing any new application, like which involves 3rd party modules etc. Though these exceed the range of kdump, however a simple test notification is good to have for now. Design ====== Kdump currently will check any relating files/fs/drivers modified before determine if initrd should rebuild when (re)start. A rebuild is an indicator of modification, so kdump need to be tested. This will clear the test status specified in $KDUMP_STATUS. Kdump test check will happen at "kdumpctl (re)start/status", and will report the tested/untested status to users. A tested status indicates previously there was a vmcore successfully generated based on the current env, so it is more likely a vmcore will be generated later when real crash happens. $KDUMP_STATUS is used for recording the newest vmcore and the test status. The format will be like: root@1.2.3.4:/var/crash 127.0.0.1-2024-05-01-15:54:29/vmcore 1714550071 untested Which means, the vmcore saved at this path, with this timestamp is the newest one, and the kdump is not tested. If later another vmcore in the same path been found, with larger(newer) timestamp. The newer vmcore will be updated into $KDUMP_STATUS, and the status will be marked as tested. (Note: There is a premise the newer vmcore is generated by the current machine. If not then the kdump test status is incorrect, see the following concurrent test case: machine1: machine2: start test checking start test checking | | V V get the newest vmcore get the newest vmcore | | V V notice user untested notice user untested | | V V expect user test kdump <-- generate vmcore by generate a vmcore | | V V ... start test checking | V find a newer vmcore | V update $KDUMP_STATUS and notice user tested <-- wrong test status In order to differentiate vmcores and corresponding machine, ip address is not reliable, like ssh dump through a NAT network. Extra code will be used to implementing this feature. Besides personally I think concurrent kdump test on multi-machines is rare. So only serial kdump test is supported for now.) The detailed updating/checking rules can be found in check_kdump_tested(). Signed-off-by: Tao Liu --- kdumpctl | 137 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/kdumpctl b/kdumpctl index 30eb27dd..8c0bdb15 100755 --- a/kdumpctl +++ b/kdumpctl @@ -16,6 +16,7 @@ KDUMP_INITRD="" TARGET_INITRD="" #kdump shall be the default dump mode DEFAULT_DUMP_MODE="kdump" +KDUMP_STATUS="/var/crash/kdump.status" standard_kexec_args="-d -p" @@ -189,6 +190,8 @@ rebuild_initrd() else rebuild_kdump_initrd fi + + clear_kdump_test_status } #$1: the files to be checked with IFS=' ' @@ -1063,6 +1066,7 @@ start() start_dump || return dinfo "Starting kdump: [OK]" + check_kdump_tested return 0 } @@ -1668,6 +1672,137 @@ _should_reset_crashkernel() { [[ $(kdump_get_conf_val auto_reset_crashkernel) != no ]] && systemctl is-enabled kdump &> /dev/null } +clear_kdump_test_status() +{ + echo > $KDUMP_STATUS +} + +# Get the latest dump's info from the dump target in the format: +# /var/crash 127.0.0.1-2024-05-01-15:54:29/vmcore 1714550071 +# +# return: 0 on success, 1 on failure +get_latest_dump() +{ + local _latest_record _target + local _path=$(get_save_path) + if is_raw_dump_target; then + dinfo "Unable to check for raw dump. Manual verification of vmcore collection required." + return 1 + elif is_ssh_dump_target; then + _latest_record=$(ssh -i "${OPT[sshkey]}" -o BatchMode=yes "${OPT[_target]}" \ + "cd $_path && \ + find . -name "vmcore" -or -name "vmcore.flat" \ + | xargs stat -c %n\ %Y 2> /dev/null \ + | sort -k2 | sed 's|^\./||' \ + | tail -1") + _target=${OPT[_target]}:$_path + else + if is_nfs_dump_target; then + if ! is_mounted ${OPT[_target]}; then + dinfo "The dump target ${OPT[_target]} is not mounted, unable to check for vmcores." + return 1 + fi + fi + _mnt=$(get_mntpoint_from_target "${OPT[_target]}") + _latest_record=$(cd $_mnt/$_path && \ + find . -name "vmcore" -or -name "vmcore-incomplete" \ + | xargs stat -c %n\ %Y 2> /dev/null \ + | sort -k2 | sed 's|^\./||' \ + | tail -1) + _target=${OPT[_target]}$_path + fi + echo "$_target $_latest_record" + return 0 +} + +# Check & Report if current kdump is tested +# The simplified logic is follows: +# +# A) If $KDUMP_STATUS not exist, then treat kexec-tools as fresh install by +# finding the latest vmcore in the host as specified by kdump.conf, record +# it into $KDUMP_STATUS, and mark & report this as untested. +# +# B) If $KDUMP_STATUS exist, then treat kexec-tools as already installed, and +# finding the latest vmcore in the host as specified by kdump.conf, +# a) If host is different from $KDUMP_STATUS's record, then update the vmcore +# into $KDUMP_STATUS, mark & report this as untested. +# b) If host is the same as $KDUMP_STATUS's record, +# 1) If the timestamp of the vmcore is equal or smaller(older) than +# $KDUMP_STATUS's record, then only report the status of $KDUMP_STATUS's +# record. +# 2) If the timestamp of the vmcore is larger(newer) than $KDUMP_STATUS's +# record, then update the vmcore into $KDUMP_STATUS, mark & report this +# as tested. +# +# host is a string like "/var/crash" for local dump, "root@1.2.3.4:/var/crash" +# for ssh dump, "1.2.3.4:/var/crash" for nfs dump. +check_kdump_tested() +{ + local _record_of_current _current_core_host _current_core_file + local _current_core_timestamp _current_core_date + local _record_of_kdstatus _kdstatus_core_host _kdstatus_core_file + local _kdstatus_core_timestamp _kdstatus_test_status + local _ret + + _record_of_current=$(get_latest_dump) + if [[ $? -ne 0 ]]; then + dwarn "Test fail: Due to previous error." + return + fi + # There may be no vmcores find by get_latest_dump + if [[ $(echo $_record_of_current | tr ' ' '\n' | wc -l) -lt 3 ]]; then + _record_of_current="dummy_host dummy_vmcore 0" + fi + _current_core_host=$(echo $_record_of_current | awk '{print $1}') + _current_core_file=$(echo $_record_of_current | awk '{print $2}') + _current_core_timestamp=$(echo $_record_of_current | awk '{print $3}') + _current_core_date=$(echo "@"$_current_core_timestamp | xargs date -d 2> /dev/null) + + if [[ ! $_current_core_file =~ .*vmcore(\.flat)?$ ]]; then + dwarn "Test fail: Latest vmcore saving is incomplete." + return + fi + + if [[ ! -s $KDUMP_STATUS ]]; then + echo "$_record_of_current untested" > $KDUMP_STATUS + dwarn "Test fail: NOT tested yet" + return + fi + + _record_of_kdstatus=$(cat $KDUMP_STATUS) + _kdstatus_core_host=$(echo $_record_of_kdstatus | awk '{print $1}') + _kdstatus_core_file=$(echo $_record_of_kdstatus | awk '{print $2}') + _kdstatus_core_timestamp=$(echo $_record_of_kdstatus | awk '{print $3}') + _kdstatus_core_date=$(echo "@"$_kdstatus_core_timestamp | xargs date -d 2> /dev/null) + _kdstatus_test_status=$(echo $_record_of_kdstatus | awk '{print $4}') + + if [[ $_kdstatus_core_host == "dummy_host" ]]; then + if [[ $_current_core_host == "dummy_host" ]]; then + dwarn "Test fail: NOT tested yet" + else + echo "$_record_of_current tested" > $KDUMP_STATUS + dinfo "Test success: Last tested on $_current_core_date" + fi + return + fi + + if [[ $_kdstatus_core_host == $_current_core_host ]]; then + if [[ $_current_core_timestamp -gt $_kdstatus_core_timestamp ]]; then + echo "$_record_of_current tested" > $KDUMP_STATUS + dinfo "Test success: Last tested on $_current_core_date" + else + if [[ $_kdstatus_test_status == "tested" ]]; then + dinfo "Test success: Last tested on $_kdstatus_core_date" + else + dwarn "Test fail: NOT tested yet" + fi + fi + else + echo "$_record_of_current untested" > $KDUMP_STATUS + dwarn "Test fail: NOT tested yet" + fi +} + main() { # Determine if the dump mode is kdump or fadump @@ -1699,6 +1834,8 @@ main() EXIT_CODE=3 ;; esac + parse_config || exit 1 + check_kdump_tested exit $EXIT_CODE ;; reload)