From 7885a3e42fb32cf26f3a92c735adf8c91c0a2b6b Mon Sep 17 00:00:00 2001 From: Tao Liu Date: Mon, 20 May 2024 11:07:36 +0800 Subject: [PATCH] Introduce vmcore creation notification to kdump Motivation ========== People may forget to recheck to ensure kdump works, which as a result, a possibility of no vmcores generated after a real system crash. It is unexpected for kdump. It is highly recommended people to recheck kdump after any system modification, such as: a. after kernel patching or whole yum update, as it might break something on which kdump is dependent, maybe due to introduction of any new bug etc. b. after any change at hardware level, maybe storage, networking, firmware upgrading etc. c. after implementing any new application, like which involves 3rd party modules etc. Though these exceed the range of kdump, however a simple vmcore creation status notification is good to have for now. Design ====== Kdump currently will check any relating files/fs/drivers modified before determine if initrd should rebuild when (re)start. A rebuild is an indicator of such modification, and kdump need to be rechecked. This will clear the vmcore creation status specified in $VMCORE_CREATION_STATUS. Vmcore creation check will happen at "kdumpctl (re)start/status", and will report the creation success/fail status to users. A "success" status indicates previously there has been a vmcore successfully generated based on the current env, so it is more likely a vmcore will be generated later when real crash happens; A "fail" status indicates previously there was no vmcore generated, or has been a vmcore creation failed based on current env. User should check the 2nd kernel log or the kexec-dmesg.log for the failing reason. $VMCORE_CREATION_STATUS is used for recording the vmcore creation status of the current env. The format will be like: success 1718682002 Which means, there has been a vmcore generated successfully at this timestamp for the current env. Usage ===== [root@localhost ~]# kdumpctl restart kdump: kexec: unloaded kdump kernel kdump: Stopping kdump: [OK] kdump: kexec: loaded kdump kernel kdump: Starting kdump: [OK] kdump: Fail: No vmcore creation test performed! [root@localhost ~]# echo c > /proc/sysrq-trigger [root@localhost ~]# kdumpctl status kdump: Kdump is operational kdump: Success: Last successful vmcore creation on Tue Jun 18 16:39:10 CST 2024 [root@localhost ~]# kdumpctl restart kdump: kexec: unloaded kdump kernel kdump: Stopping kdump: [OK] kdump: kexec: loaded kdump kernel kdump: Starting kdump: [OK] kdump: Success: Last successful vmcore creation on Tue Jun 18 16:39:10 CST 2024 The notification for kdumpctl (re)start/status can be disabled by setting VMCORE_CREATION_NOTIFICATION in /etc/sysconfig/kdump Signed-off-by: Tao Liu --- dracut-kdump.sh | 35 ++++++++++++++++++++++++++--------- gen-kdump-sysconfig.sh | 4 ++++ kdump-lib-initramfs.sh | 24 ++++++++++++++++++++++++ kdumpctl | 29 +++++++++++++++++++++++++++++ mkdumprd | 5 ++--- 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/dracut-kdump.sh b/dracut-kdump.sh index 7b3ad7a7..50ddb253 100755 --- a/dracut-kdump.sh +++ b/dracut-kdump.sh @@ -237,16 +237,8 @@ save_opalcore_fs() return 0 } -dump_to_rootfs() +try_mount_sysroot() { - - if [ "$(systemctl status dracut-initqueue | sed -n "s/^\s*Active: \(\S*\)\s.*$/\1/p")" = "inactive" ]; then - dinfo "Trying to bring up initqueue for rootfs mount" - systemctl start dracut-initqueue - fi - - dinfo "Clean up dead systemd services" - systemctl cancel dinfo "Waiting for rootfs mount, will timeout after 90 seconds" systemctl start --no-block sysroot.mount @@ -258,6 +250,23 @@ dump_to_rootfs() if ! is_mounted /sysroot; then derror "Failed to mount rootfs" + return 1 + fi + return 0 +} + +dump_to_rootfs() +{ + + if [ "$(systemctl status dracut-initqueue | sed -n "s/^\s*Active: \(\S*\)\s.*$/\1/p")" = "inactive" ]; then + dinfo "Trying to bring up initqueue for rootfs mount" + systemctl start dracut-initqueue + fi + + dinfo "Clean up dead systemd services" + systemctl cancel + + if ! try_mount_sysroot; then return fi @@ -312,11 +321,19 @@ do_final_action() do_dump() { + if ! try_mount_sysroot; then + return 1 + fi + set_vmcore_creation_status 'clear' "/sysroot/$VMCORE_CREATION_STATUS" + eval $DUMP_INSTRUCTION _ret=$? if [ $_ret -ne 0 ]; then + set_vmcore_creation_status 'fail' "/sysroot/$VMCORE_CREATION_STATUS" derror "saving vmcore failed" + else + set_vmcore_creation_status 'success' "/sysroot/$VMCORE_CREATION_STATUS" fi return $_ret diff --git a/gen-kdump-sysconfig.sh b/gen-kdump-sysconfig.sh index 1a2cd920..0bd09f01 100755 --- a/gen-kdump-sysconfig.sh +++ b/gen-kdump-sysconfig.sh @@ -47,6 +47,10 @@ KDUMP_IMG="vmlinuz" #What is the images extension. Relocatable kernels don't have one KDUMP_IMG_EXT="" +# Enable vmcore creation notification by default, disable by setting +# VMCORE_CREATION_NOTIFICATION="" +VMCORE_CREATION_NOTIFICATION="yes" + # Logging is controlled by following variables in the first kernel: # - @var KDUMP_STDLOGLVL - logging level to standard error (console output) # - @var KDUMP_SYSLOGLVL - logging level to syslog (by logger command) diff --git a/kdump-lib-initramfs.sh b/kdump-lib-initramfs.sh index f7b58a2c..fceade09 100755 --- a/kdump-lib-initramfs.sh +++ b/kdump-lib-initramfs.sh @@ -9,6 +9,7 @@ KDUMP_CONFIG_FILE="/etc/kdump.conf" FENCE_KDUMP_CONFIG_FILE="/etc/sysconfig/fence_kdump" FENCE_KDUMP_SEND="/usr/libexec/fence_kdump_send" LVM_CONF="/etc/lvm/lvm.conf" +VMCORE_CREATION_STATUS="/var/crash/vmcore-creation.status" # Read kdump config in well formated style kdump_read_conf() @@ -171,3 +172,26 @@ kdump_get_ip_route_field() { echo "$1" | sed -n -e "s/^.*\<$2\>\s\+\(\S\+\).*$/\1/p" } + +# $1: success/fail/clear +# $2: status_file +set_vmcore_creation_status() +{ + _status=$1 + _status_file=$2 + _dir=${_status_file%%${_status_file##*/}} + + [[ -d "$_dir" ]] || mkdir -p "$_dir" + + case "$_status" in + success | fail) + echo "$_status $(date +%s)" > "$_status_file" + ;; + clear) + rm -f "$_status_file" + ;; + *) + return + esac + sync -f "$_dir" +} \ No newline at end of file diff --git a/kdumpctl b/kdumpctl index 30eb27dd..0011f282 100755 --- a/kdumpctl +++ b/kdumpctl @@ -189,6 +189,8 @@ rebuild_initrd() else rebuild_kdump_initrd fi + + set_vmcore_creation_status 'clear' "$VMCORE_CREATION_STATUS" } #$1: the files to be checked with IFS=' ' @@ -1063,6 +1065,7 @@ start() start_dump || return dinfo "Starting kdump: [OK]" + check_vmcore_creation_status return 0 } @@ -1668,6 +1671,31 @@ _should_reset_crashkernel() { [[ $(kdump_get_conf_val auto_reset_crashkernel) != no ]] && systemctl is-enabled kdump &> /dev/null } +check_vmcore_creation_status() +{ + local _record _status _timestamp _status_date + + if [[ -z "$VMCORE_CREATION_NOTIFICATION" ]]; then + return + fi + + if [[ ! -s $VMCORE_CREATION_STATUS ]]; then + dwarn "Fail: No vmcore creation test performed!" + return + fi + + _record=$(cat "$VMCORE_CREATION_STATUS") + _status=$(echo $_record | awk '{print $1}') + _timestamp=$(echo $_record | awk '{print $2}') + _status_date=$(echo "@"$_timestamp | xargs date -d 2> /dev/null) + + if [[ "$_status" == "success" ]]; then + dinfo "Success: Last successful vmcore creation on $_status_date" + else + dwarn "Fail: Last unsuccessful vmcore creation on $_status_date" + fi +} + main() { # Determine if the dump mode is kdump or fadump @@ -1699,6 +1727,7 @@ main() EXIT_CODE=3 ;; esac + check_vmcore_creation_status exit $EXIT_CODE ;; reload) diff --git a/mkdumprd b/mkdumprd index 27eed5e3..07b88f89 100644 --- a/mkdumprd +++ b/mkdumprd @@ -403,9 +403,8 @@ fi # parameter available in fadump case. So, find a way to fix that first # before removing this check. if ! is_fadump_capable; then - # The 2nd rootfs mount stays behind the normal dump target mount, - # so it doesn't affect the logic of check_dump_fs_modified(). - is_dump_to_rootfs && add_mount "$(to_dev_name "$(get_root_fs_device)")" + # Always mount rootfs since we will create VMCORE_CREATION_STATUS file + add_mount "$(to_dev_name "$(get_root_fs_device)")" add_dracut_arg "--no-hostonly-default-device"