diff --git a/00-default.conf b/00-default.conf index b77d043..079974a 100644 --- a/00-default.conf +++ b/00-default.conf @@ -9,33 +9,55 @@ # file, it's required to re-create the kdump minimal initramfs by running: # /usr/lib/kdump/kdump-load.sh initrd # - -# Mount-related options - the DEVNODE points to the /home directory link; -# this is used to derive the numerical devnode for kdump, since the link -# is not present so early in the system boot. The KDUMP_FOLDER will be -# created if doesn't exist. The KDUMP_MNT is just a temporary file that -# carries the mounted folder path across boot-time scripts. - +# +# Pstore-RAM settings +# If USE_PSTORE_RAM is set to 1, Kdump won't be loaded. Instead, the Pstore +# RAM backend will be configured. In order to have success, this operation +# relies in having an available RAM buffer on /proc/iomem with at least +# PSTORE_MEM_AMOUNT (decimal, in MB) in size. Also, kernel must be able to +# allocate a contiguous memory amount of PSTORE_RECORD_SZ (decimal, MB). +USE_PSTORE_RAM=1 +PSTORE_MEM_AMOUNT=5242880 +PSTORE_RECORD_SZ=2097152 +# +# +# Mount-related options +# MOUNT_DEVNODE is the desired mount-point unique link in the /dev/disk +# directory link; this is used to derive the numerical devnode for kdump, +# since the link is not present so early in the system boot. MOUNT_FOLDER +# should live in the filesystem of the devnode (and will be created if +# doesn't exist); this is the place in which both pstore and kdump logs +# will be stored, as well as the kdump initrd and some ancillary data. +# Finally, MNT_TMP is just a temporary file that carries the full path +# of the mounted folder across boot-time scripts. MOUNT_DEVNODE="/dev/disk/by-partsets/shared/home" -KDUMP_FOLDER="/.steamos/offload/var/kdump" -KDUMP_MNT="/tmp/kdump.mnt" - -# Kdump controlling settings - for now we don't have network/iscsi dumps, only -# local storage dumps. If FULL_COREDUMP is !=0, we collect a full compressed -# vmcore, which might require a lot of disk space. The *_CMD settings refer -# to tunings on makedumpfile - we rely on zstd compression and maximum page -# exclusion for the full vmcore, mimic'ing Debian kdump. Finally, GRUB_AUTOSET -# determines if the kdump tooling should try to automatically set grub.cfg in -# order to reserve memory for kdump (if needed, i.e., when pstore is not in -# use) - keep it as '1' for enabling the mechanism (this is the default). - +MOUNT_FOLDER="/.steamos/offload/var/kdump" +MNT_TMP="/tmp/.kdump_load.mnt" +# +# +# Kdump controlling settings +# Currently we only do local storage log collection (no network/iscsi dumps). +# If FULL_COREDUMP is !=0, we collect a full compressed vmcore, which might +# require a lot of disk space. The MAKEDUMPFILE_*_CMD settings refer to +# tunings on makedumpfile - we rely on zstd compression and maximum page +# exclusion for the full vmcore, mimic'ing Debian kdump FULL_COREDUMP=0 MAKEDUMPFILE_COREDUMP_CMD="-z -d 31" MAKEDUMPFILE_DMESG_CMD="--dump-dmesg" +# +# +# GRUB-related settings +# GRUB_AUTOSET determines if the pstore/kdump tooling should try to +# automatically set grub.cfg in order to reserve memory for kdump (if +# needed, i.e., when pstore is not in use) - keep it as '1' for enabling +# the mechanism (this is the default). GRUB_BOOT_FILE is the full path of +# grub.cfg file - this is used in the config update step plus for syncing +# purposes. GRUB_CFG_FILE is the config file we edit in order to add/remove +# the crashkernel parameter. Finally, GRUB_CMDLINE holds the change to be +# made in the grub config file, specially the crashkernel memory reservation +# (notice that a trailing space is required in this line, so we avoid +# messing with other kernel parameters). GRUB_AUTOSET=1 - -# Pstore-RAM setting - if enabled, Kdump won't be loaded, instead the Pstore -# RAM backend will be configured. In order to have success, this operation -# relies in having an available RAM buffer on /proc/iomem with at least 5MiB -# in size. -USE_PSTORE_RAM=1 +GRUB_BOOT_FILE="/efi/EFI/steamos/grub.cfg" +GRUB_CFG_FILE="/etc/default/grub" +GRUB_CMDLINE="crashkernel=256M crash_kexec_post_notifiers " diff --git a/kdump-collect.sh b/kdump-collect.sh index 488f277..933c6a4 100644 --- a/kdump-collect.sh +++ b/kdump-collect.sh @@ -23,7 +23,7 @@ done VMCORE="/proc/vmcore" KDUMP_TIMESTAMP=$(date -u +"%Y%m%d%H%M") -KDUMP_FOLDER="/kdump_path/${KDUMP_FOLDER}/crash/${KDUMP_TIMESTAMP}" +KDUMP_FOLDER="/kdump_path/${MOUNT_FOLDER}/crash/${KDUMP_TIMESTAMP}" # Bail out in case we don't have a vmcore, i.e. either we're not kdumping # or something is pretty wrong and we wouldn't be able to progress. diff --git a/kdump-load.sh b/kdump-load.sh index 017a910..2477ffe 100644 --- a/kdump-load.sh +++ b/kdump-load.sh @@ -9,40 +9,46 @@ # configures the Pstore-RAM mechanism. If the proper parameters are passed # also, either it creates the minimal kdump initramfs for the running kernel # or removes all the previously created ones. Since it runs on boot time, -# avoid failing here to not risk a boot hang. +# extra care is required to avoid boot hangs. # # This function has 2 purposes: if 'kdump' is passed as argument and we don't # have crashkernel memory reserved, we edit grub config file and recreate -# grub.cfg, so next boot has it reserved; in this case, we also bail-out, +# grub.cfg, so next boot has it reserved; in this case, we also bail-out, # since kdump can't be loaded anyway. # # If 'pstore' is passsed as argument, we try to unset crashkernel iff it's # already set AND the pattern in grub config is the one added by us - if the # users set crashkernel themselves, we don't mess with that. grub_update() { - GRUBCFG="/etc/default/grub" CRASHK="$(cat /sys/kernel/kexec_crash_size)" - SED_ADD="s/^GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"crashkernel=192M crash_kexec_post_notifiers /g" + SED_ADD="s/^GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"${GRUB_CMDLINE}/g" if [ "${GRUB_AUTOSET}" -eq 1 ]; then if [ "$1" = "kdump" ] && [ "${CRASHK}" -eq 0 ]; then - sed -i "${SED_ADD}" "${GRUBCFG}" - update-grub 1>/dev/null - sync "/boot/grub/grub.cfg" 2>/dev/null - sync "/efi/EFI/steamos/grub.cfg" 2>/dev/null + sed -i "${SED_ADD}" "${GRUB_CFG_FILE}" - logger "kdump: kexec cannot work, no reserved memory in this boot..." + if ! grub-mkconfig -o "${GRUB_BOOT_FILE}" 1>/dev/null; then + logger "kdump: failed to execute command \"${GRUB_CMD}\"" + exit 1 + fi + sync "${GRUB_BOOT_FILE}" 2>/dev/null + + logger "kdump: kexec won't succeed, no reserved memory in this boot..." logger "kdump: but we automatically set crashkernel for next boot." exit 0 # this is considered a successful run fi if [ "$1" = "pstore" ] && [ "${CRASHK}" -ne 0 ]; then - sed -i "s/\"crashkernel=192M crash_kexec_post_notifiers /\"/g" "${GRUBCFG}" - update-grub 1>/dev/null - sync "/boot/grub/grub.cfg" 2>/dev/null - sync "/efi/EFI/steamos/grub.cfg" 2>/dev/null - logger "kdump: clearing crashkernel memory previously set..." + sed -i "s/\"${GRUB_CMDLINE}/\"/g" "${GRUB_CFG_FILE}" + + if ! grub-mkconfig -o "${GRUB_BOOT_FILE}" 1>/dev/null; then + logger "kdump: failed to execute command \"${GRUB_CMD}\"" + exit 1 + fi + sync "${GRUB_BOOT_FILE}" 2>/dev/null + + logger "kdump: cleared crashkernel memory previously set." fi fi } @@ -50,27 +56,27 @@ grub_update() { # This function is responsible for creating the kdump initrd, either # via command-line call or in case initrd doesn't exist during kdump load. create_initrd() { - rm -f "${KDUMP_FOLDER}/kdump-initrd-$(uname -r).img" + rm -f "${MOUNT_FOLDER}/kdump-initrd-$(uname -r).img" echo "Creating the kdump initramfs for kernel \"$(uname -r)\" ..." DRACUT_NO_XATTR=1 dracut --no-early-microcode --host-only -q -m\ "bash systemd systemd-initrd systemd-sysusers modsign dbus-daemon kdump dbus udev-rules dracut-systemd base fs-lib shutdown"\ - --kver "$(uname -r)" "${KDUMP_FOLDER}/kdump-initrd-$(uname -r).img" + --kver "$(uname -r)" "${MOUNT_FOLDER}/kdump-initrd-$(uname -r).img" } # This routine performs a clean-up by deleting the old/useless remaining # kdump initrd files. cleanup_unused_initrd() { - INSTALLED_KERNELS="${KDUMP_FOLDER}/.installed_kernels" + INSTALLED_KERNELS="${MOUNT_FOLDER}/.installed_kernels" find /lib/modules/* -maxdepth 0 -type d -exec basename {} \;>"${INSTALLED_KERNELS}" - find "${KDUMP_FOLDER}"/* -name "kdump-initrd*" -type f -print0 | while IFS= read -r -d '' file + find "${MOUNT_FOLDER}"/* -name "kdump-initrd*" -type f -print0 | while IFS= read -r -d '' file do FNAME="$(basename "${file}" .img)" KVER="${FNAME#kdump-initrd-}" if ! grep -q "${KVER}" "${INSTALLED_KERNELS}" ; then - rm -f "${KDUMP_FOLDER}/${FNAME}.img" + rm -f "${MOUNT_FOLDER}/${FNAME}.img" logger "kdump: removed unused file \"${FNAME}.img\"" fi done @@ -97,42 +103,42 @@ fi # Find the proper mount point expected for kdump collection: DEVN_MOUNTED="$(findmnt "${MOUNT_DEVNODE}" -fno TARGET)" -# Create the kdump folder here, as soon as possible, given the -# importance of such directory in all kdump/pstore steps. -KDUMP_FOLDER="${DEVN_MOUNTED}/${KDUMP_FOLDER}" -mkdir -p "${KDUMP_FOLDER}" +# Create the kdump main folder here, as soon as possible, given +# the importance of such directory in all kdump/pstore steps. +MOUNT_FOLDER="${DEVN_MOUNTED}/${MOUNT_FOLDER}" +mkdir -p "${MOUNT_FOLDER}" -echo "${KDUMP_FOLDER}" > "${KDUMP_MNT}" -sync "${KDUMP_MNT}" +echo "${MOUNT_FOLDER}" > "${MNT_TMP}" +sync "${MNT_TMP}" # Notice that at this point it's required to have the full -# KDUMP_FOLDER, so this must remain after the DEVNODE operations above. +# MOUNT_FOLDER, so this must remain after the DEVNODE operations above. if [ "$1" = "initrd" ]; then create_initrd exit 0 fi if [ "$1" = "clear" ]; then - rm -f "${KDUMP_FOLDER}"/kdump-initrd-* + rm -f "${MOUNT_FOLDER}"/kdump-initrd-* exit 0 fi -# Pstore-RAM load; if it is configured via /usr/share/kdump/kdump.conf and fails +# Pstore-RAM load; if it is configured via the config files and fails # to configure pstore, we still try to load the kdump. We try to reserve -# here a 5MiB memory region. -# Notice that we assume ramoops is a module here - if built-in, we should -# properly load it through command-line parameters. +# here a ${MEM_REQUIRED} memory region. +# Notice that we assume ramoops is a module here - if built-in, users +# should properly load it through command-line parameters. if [ "${USE_PSTORE_RAM}" -eq 1 ]; then - MEM_REQUIRED=5242880 # 5MiB - RECORD_SIZE=0x200000 # 2MiB + MEM_REQUIRED="${PSTORE_MEM_AMOUNT}" + RECORD_SIZE="${PSTORE_RECORD_SZ}" RANGE=$(grep "RAM buffer" /proc/iomem | head -n1 | cut -f1 -d\ ) MEM_END=$(echo "$RANGE" | cut -f2 -d-) MEM_START=$(echo "$RANGE" | cut -f1 -d-) MEM_SIZE=$(( 16#${MEM_END} - 16#${MEM_START} )) - if [ ${MEM_SIZE} -ge ${MEM_REQUIRED} ]; then - if modprobe ramoops mem_address=0x"${MEM_START}" mem_size=${MEM_REQUIRED} record_size=${RECORD_SIZE}; then + if [ ${MEM_SIZE} -ge "${MEM_REQUIRED}" ]; then + if modprobe ramoops mem_address=0x"${MEM_START}" mem_size="${MEM_REQUIRED}" record_size="${RECORD_SIZE}"; then # If Pstore is set, update grub.cfg to avoid reserving crashkernel memory. logger "kdump: pstore-RAM was loaded successfully" cleanup_unused_initrd @@ -159,7 +165,7 @@ VMLINUX="$(grep -o 'BOOT_IMAGE=[^ ]*' /proc/cmdline)" # In case we don't have a valid initrd, for some reason, try creating # one before loading kdump (or else it will fail). -INITRD_FNAME="${KDUMP_FOLDER}/kdump-initrd-$(uname -r).img" +INITRD_FNAME="${MOUNT_FOLDER}/kdump-initrd-$(uname -r).img" if [ ! -s "${INITRD_FNAME}" ]; then create_initrd fi diff --git a/save-dumps.sh b/save-dumps.sh index 6d86887..7060c25 100644 --- a/save-dumps.sh +++ b/save-dumps.sh @@ -25,16 +25,16 @@ if [ ${HAVE_CFG_FILES} -eq 0 ]; then exit 1 fi -KDUMP_MAIN_FOLDER="$(cat "${KDUMP_MNT}")" -rm -f "${KDUMP_MNT}" +MAIN_FOLDER="$(cat "${MNT_TMP}")" +rm -f "${MNT_TMP}" -if [ ! -d "${KDUMP_MAIN_FOLDER}" ]; then - logger "kdump: invalid folder (${KDUMP_MAIN_FOLDER}) - aborting..." +if [ ! -d "${MAIN_FOLDER}" ]; then + logger "kdump: invalid folder (${MAIN_FOLDER}) - aborting..." exit 1 fi LOGS_FOUND=0 -KDUMP_TMP_FOLDER="${KDUMP_MAIN_FOLDER}/.tmp" +KDUMP_TMP_FOLDER="${MAIN_FOLDER}/.tmp" # Use UTC timezone to match kdump collection CURRENT_TSTAMP=$(date -u +"%Y%m%d%H%M") @@ -67,7 +67,7 @@ if [ "${PSTORE_CNT}" -ne 0 ]; then fi # Now, we proceed the same way if there are kdump data. -KDUMP_CRASH_FOLDER="${KDUMP_MAIN_FOLDER}/crash" +KDUMP_CRASH_FOLDER="${MAIN_FOLDER}/crash" KDUMP_CNT=$(find "${KDUMP_CRASH_FOLDER}"/* -type d 2>/dev/null | wc -l) if [ "${KDUMP_CNT}" -ne 0 ]; then @@ -112,10 +112,10 @@ fi # If we have pstore and/or kdump logs, let's process them... -KDUMP_LOGS_FOLDER="${KDUMP_MAIN_FOLDER}/logs" +LOGS_FOLDER="${MAIN_FOLDER}/logs" if [ ${LOGS_FOUND} -ne 0 ]; then - mkdir -p "${KDUMP_LOGS_FOLDER}" + mkdir -p "${LOGS_FOLDER}" # First we collect some more info, like DMI data, os-release, etc; DMI_FNAME="${KDUMP_TMP_FOLDER}/dmidecode.${CURRENT_TSTAMP}" @@ -131,7 +131,7 @@ if [ ${LOGS_FOUND} -ne 0 ]; then # Create the dump compressed pack. LOG_FNAME="kdump-${CURRENT_TSTAMP}.zip" - LOG_FNAME="${KDUMP_LOGS_FOLDER}/${LOG_FNAME}" + LOG_FNAME="${LOGS_FOLDER}/${LOG_FNAME}" zip -9 -jq "${LOG_FNAME}" "${KDUMP_TMP_FOLDER}"/* 1>/dev/null sync "${LOG_FNAME}" 2>/dev/null @@ -139,7 +139,7 @@ if [ ${LOGS_FOUND} -ne 0 ]; then logger "kdump: couldn't create the compressed log archive" logger "kdump: check folder \"${KDUMP_TMP_FOLDER}\" for logs" else - logger "kdump: logs saved in \"${KDUMP_LOGS_FOLDER}\"" + logger "kdump: logs saved in \"${LOGS_FOLDER}\"" fi fi