From fae5982a5049c5f1d9ba0ac4453ca59d72245cd7 Mon Sep 17 00:00:00 2001 From: "Guilherme G. Piccoli" Date: Wed, 2 Feb 2022 16:43:41 -0300 Subject: [PATCH] submit_report.sh, kdump.etc: Add log submission mechanism Finally we have a functional mechanism to upload the crash logs to Valve servers (special thanks to TonyP for the API help). Documentation is present in the README.MD as usual. NOTE: worth to reinforce here what was alread mentioned in the README: kdump-steamos doesn't perform any significant validation against malicious usage of the log submission mechanism, like DDoS, submitting a malicious ZIP binary, a very huge file, etc. All of this is expected to be handled by the server side. Signed-off-by: Guilherme G. Piccoli --- PKGBUILD | 8 +-- README.md | 110 ++++++++++++++++++++++++++++++------- kdump.etc | 7 ++- submit_report.sh | 139 ++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 231 insertions(+), 33 deletions(-) diff --git a/PKGBUILD b/PKGBUILD index 83d54fd..5243efa 100644 --- a/PKGBUILD +++ b/PKGBUILD @@ -7,7 +7,7 @@ pkgname=kdump-steamos pkgver=0.4 pkgrel=1 pkgdesc="Kdump scripts to collect vmcore/dmesg in a small dracut-based initramfs" -depends=('dmidecode' 'dracut' 'kexec-tools' 'makedumpfile' 'systemd' 'zip' 'zstd') +depends=('curl' 'dmidecode' 'dracut' 'jq' 'kexec-tools' 'makedumpfile' 'systemd' 'zip' 'zstd') arch=('x86_64') license=('GPL2') install=kdump-steamos.install @@ -25,13 +25,13 @@ source=('20-kdump-steamos.conf' sha256sums=('dbedff54addfb5dce51614c73df04c90fca9f27d0d3a690243259ccbbfcca07c' '2514f79a496f76af847e262eadd55a5c2f8d95375cc513efa8cadd4cd98fe1d2' - 'd0ac5e7e38fa1d3355eacdf70188483456f53d3e2b18cd161dea3df87b0b8f9c' + 'fca89993e6ca76ec14f9a8e6761bec521903e2743ab9e05bfe2cd2b77e5e89f9' '8a556a9ebbda88dfd29b9620a0f2e7dbea19cd5fc019eb5dc4ebf7c80e4bf238' '06b38bd9f09da5fb22a765b6f1945fc349cc5f9d13cd32c9218b9b60b40a9010' '12a9124b907f208471ba7aaac0f3261cbbd34a168cce3260fa9e7793994beebd' '26bc2b64af0d468f050c0e0dd9e2053176d56886edad9146bc495797bf2c5810' - '0b602c27a5ff30c4fdb944135bbd9309225f31fb4ce354b7b07a03ce56b55531' - '8f974b51f6fcf5ec4ae8bb9d585390d4354cf8d99ae98e0efe9484f08abb1949' + '97d055f9e6e66cc4168767f5b51ff8e728c6935d270a997cf5b082ddd2d30634' + 'b1a4a0f77cae137d3f5155a0fcc4b9d1e1b7ffd0089adb06b6bb21e71c751e27' 'cbb207ecc0f6bacefbeed41f0d4910daac6500ac2345366e1f95f09a7653c65a') package() { diff --git a/README.md b/README.md index 4bd0f61..de6fbcc 100644 --- a/README.md +++ b/README.md @@ -21,30 +21,33 @@ # # # 1. Install the package with pacman if not available in your image - there's -# a pre-built binary package in this gitlab; to check if it's already installed -# look the pacman installed package list. Also, be sure the systemd service was -# properly loaded by checking 'systemctl status kdump-steamos.service'. +# a prebuilt binary package in this gitlab; to check if it's already installed +# look the pacman installed package list. Also, be sure the systemd service +# was properly loaded by checking 'systemctl status kdump-steamos.service'. # -# 2. Only the dmesg is collected, and by default this happens via the Pstore -# mechanism, i.e., no extra memory should be reserved and no GRUB change is -# required. If 'lsmod' shows "ramoops", then Pstore is in use. +# 2. In a crash event, the dmesg log is collected, and by default this happens +# via the Pstore mechanism, i.e., no extra memory should be reserved and no +# GRUB change is required. If 'lsmod' shows "ramoops", then Pstore is in use. +# Besides the dmesg with some extra information (like tasks running, memory +# usage on crash, etc), more logs are collected like the image build version, +# running kernel version and dmidecode. # # 3. The logs are stored in a ZIP file at "/home/.steamos/offload/var/kdump/"; -# besides the dmesg with some extra information, the image build version, -# running kernel version and dmidecode are stored in this ZIP file as well. -# This file is named as: "steamos-SERIAL-STEAM_USER.timestamp.zip", where -# SERIAL is the machine serial (from dmidecode), STEAM_USER is the Steam -# account name (based on the last logged Steam user) and timestamp tz is UTC. +# if this ZIP file was successfully submitted to Valve servers, this file is +# then moved into the sub-folder "sent_logs/". This file is named as: +# "steamos-SERIAL-STEAM_USER.timestamp.zip", where SERIAL is the machine +# serial (from dmidecode), STEAM_USER is the Steam account name (based on the +# last logged Steam user) and timestamp tz is UTC. # # 4. (IMPORTANT) Please, test the infrastructure in order to see if a dummy # crash log is collected before using it to try debugging complex issues. # In order to do that, login to a shell and execute, as root user: # 'echo 1 > /proc/sys/kernel/sysrq ; echo c > /proc/sysrq-trigger' # -# This action will trigger a dummy crash and reboot the system; check if there -# is a ZIP file with the crash logs in the directory described in (3). +# This action will trigger a dummy crash and reboot the system; check if +# there is a ZIP file with the crash logs in the directory described in (3). # -# 5. Some tunnings are available at "/etc/default/kdump"; for example users +# 5. Some tunings are available at "/etc/default/kdump"; for example users # can choose Kdump instead of Pstore (USE_PSTORE_RAM), and if using Kdump, # collect the full vmcore (FULL_COREDUMP). The vmcore is not stored in the # ZIP file, but it's saved in "/home/.steamos/offload/var/kdump/crash/". @@ -52,9 +55,13 @@ # needed in GRUB cmdline: "crashkernel=192M crash_kexec_post_notifiers" and # a regular reboot is necessary. # +# 6. Error and succeeding messages are sent to systemd journal, so running +# 'journalctl | grep kdump' would hopefully bring some information. Also, +# the ZIP file collected is automatically submitted to Valve servers; see +# below under DETAILS/LOG SUBMISSION for API details, decisions made, etc. +# # # ############################## DETAILS ################################## -# # CAVEATS / INSTRUCTIONS # ########################################################################### # (a) Currently, we don't automatically edit GRUB config; see TODO (1) below. @@ -91,9 +98,8 @@ # (1) We'd like to be able to automatically edit GRUB and recreate its config # file - implementation tests are ongoing. # -# (2) The log submission mechanism is incomplete - we save the logs as a local -# ZIP file (as discussed in the HOW-TO), but they aren't submitted to a remote -# Valve server. There's an API in-place, so the implementation is starting. +# (2) Would be interesting to have a clean-up mechanism, to keep up to N most +# recent ZIP log files, instead of keeping all of them forever. # # (3) Hopefully we can fix/prevent the unnecessary re-creation of all initramfs # images - it happens due to our package installing files on directory @@ -101,8 +107,10 @@ # # (4) We have a "fragile" way of determining a mount point required for Kdump; # this is something to improve maybe, in order to make the Kdump more reliable. +# Also in the list of fragile things, VDF parsing is...complicated. Something +# that would be nice to improve as well. # -# (5) Pstore ramoops backend has some limitations that we're discussing with +# (5) Pstore ramoops back-end has some limitations that we're discussing with # the kernel community - right now we can only collect ONE dmesg and its # size is truncated on "record_size" bytes, not allowing a file split like # efi-pstore; thankfully we still can collect 2MiB dmesg, but hopefully we can @@ -116,4 +124,68 @@ # specified kernel, not only for the running one (which is what we do now). # Low-priority idea, easy to implement. # +# +# LOG SUBMISSION +# ########################################################################### +# The logs collected and compressed in the ZIP file are kept in the system, +# but they provide valuable data to Valve in order to determine issue in the +# field, and hopefully fix them, so users are happy. Hence, the kdump-steamos +# is capable now to submit logs to Valve servers, through an API. Below such +# API is described, but first worth to mention some assumptions / decisions +# made in the log submission mechanism: +# +# * First of all, we attempt to verify network connectivity by pinging the +# URL "steampowered.com" - quick pings (2 packets, 0.5s between each one) +# are attempted, but if after 99 of such pings network is considered not +# not reliable, the log submission is aborted, but the ZIP file is kept +# locally of course. +# +# * The 'curl' tool is used to submit the requests to Valve servers; for +# that, some temporary files named ".curl_XXX" are saved in the kdump +# folder - mentioned in the point (3) above. These files are deleted +# if the log submission mechanism works fine, or else they're currently +# kept for debug purposes, along with a new ".curl_err" file. +# +# * It is assumed that any throttling / anti-DoS mechanism comes from the +# server portion, so the kdump-steamos doesn't perform any significant +# validations with this respect, only basic correctness validations. +# +# +# => The API details: it works by a first POST request to Valve servers, +# which, when succeed, returns 3 main components in the response. We use +# these values to perform a PUT request with the ZIP compressed file, and +# finally a last POST request is necessary to finish the transaction. The +# POST requests' URL is present in "/etc/default/kdump". +# Below, the specific format of such requests: +# +# The first POST takes the following fields: +# +# steamid = user Steam ID, based on the latest Steam logged user; +# have_dump_file = 0/1 - should be 1 when sending a ZIP file; +# dump_file_size = the ZIP file size, in bytes; +# product = "holo" (hard-coded for now); +# build = the SteamOS build ID, from '/etc/os-release' file; +# version = running kernel version; +# platform = "linux" (hard-coded for now); +# crash_time = the timestamp (epoch) of log collection/submission; +# stack = a really concise call trace summary, only functions/addrs; +# note = summary of the dmesg crash info, specifically a full stack trace; +# format = "json" (hard-coded for now). +# +# The response of a succeeding POST will have multiple fields, that can +# be split in 3 categories: +# +# PUT_URL = a new URL to be used in the PUT request; +# GID = special ID used to finish the submission process in the next POST; +# header name/value pairs = multiple pairs of name/value fields used as +# headers in the PUT request. +# +# After parsing the response, we perform a PUT request to the PUT_URL, with +# the ZIP file as a "--data-binary" component and the additional headers that +# were collected in the first POST's response. Finally, we just POST the GID +# to the finish URL ("gid=GID_NUM") and the process is terminated. +# +# Notice we heavily use 'jq' tool to parse the JSON response, so we assume +# this format is the response one and that it's not changing over time. +# ``` diff --git a/kdump.etc b/kdump.etc index 8b92cc1..7222c8d 100644 --- a/kdump.etc +++ b/kdump.etc @@ -34,7 +34,8 @@ MAKEDUMPFILE_DMESG_CMD="--dump-dmesg" # in size. USE_PSTORE_RAM=1 -# This is a log submission setting, based on Steam config files, and -# *should not* be changed, or else the log sending mechanism will be -# impaired. +# Below some log submission settings, based on Steam config files and Valve +# URLs. These settings *should not* be changed, or else the log sending +# mechanism will be impaired. LOGINVDF="/home/doorstop/.local/share/Steam/config/loginusers.vdf" +POST_URL="https://api.steampowered.com/ICrashReportService/ACTIONCrashUpload/v1" diff --git a/submit_report.sh b/submit_report.sh index 0300a1d..6697232 100755 --- a/submit_report.sh +++ b/submit_report.sh @@ -33,6 +33,7 @@ KDUMP_LOGS_FOLDER="${KDUMP_MAIN_FOLDER}/logs" # Use UTC timezone to match kdump collection CURRENT_TSTAMP=$(date -u +"%Y%m%d%H%M") +CURRENT_EPOCH=$(date +"%s") # We assume pstore is mounted by default, in this location; # if not, we get a 0 and don't loop. @@ -115,6 +116,7 @@ if [ ${LOGS_FOUND} -ne 0 ]; then fi STEAM_ACCOUNT=0 + STEAM_ID=0 if [ -s "${LOGINVDF}" ]; then # The following awk command was borrowed from: # https://unix.stackexchange.com/a/663959 @@ -128,24 +130,147 @@ if [ ${LOGS_FOUND} -ne 0 ]; then fi STEAM_ACCOUNT=$(awk -v n=${IDX} -v RS='}' 'NR==n{gsub(/.*\{\n|\n$/,""); print}' "${LOGINVDF}" | grep "AccountName" | cut -f4 -d\") + + # Get also the Steam ID, used in the POST request to Valve servers; this + # is a bit fragile, but there's no proper VDF parse tooling it seems... + LN=$(grep -n "AccountName.*${STEAM_ACCOUNT}\"" "${LOGINVDF}" | cut -f1 -d:) + LN=$((LN - 2)) + STEAM_ID=$(sed -n "${LN}p" "${LOGINVDF}" | cut -f2 -d\") break done fi # Here we collect some more info, like DMI data, os-release, etc; - # ToDo: Add Steam application / Proton / Games logs collection... + # TODO: Add Steam application / Proton / Games logs collection... dmidecode > "${KDUMP_LOGS_FOLDER}/dmidecode.${CURRENT_TSTAMP}" - grep "BUILD_ID" "/etc/os-release" | cut -f2 -d\= > "${KDUMP_LOGS_FOLDER}/build.${CURRENT_TSTAMP}" - uname -r > "${KDUMP_LOGS_FOLDER}/version.${CURRENT_TSTAMP}" + + BUILD_FNAME="${KDUMP_LOGS_FOLDER}/build.${CURRENT_TSTAMP}" + cp "/etc/os-release" "${BUILD_FNAME}" + + VERSION_FNAME="${KDUMP_LOGS_FOLDER}/version.${CURRENT_TSTAMP}" + uname -r > "${VERSION_FNAME}" + + # Before compressing the logs, save a crash summary + CRASH_SUMMARY="${KDUMP_LOGS_FOLDER}/crash_summary.${CURRENT_TSTAMP}" + SED_EXPR="/Kernel panic \-/,/Kernel Offset\:/p" + sed -n "${SED_EXPR}" "${KDUMP_LOGS_FOLDER}"/dmesg* > "${CRASH_SUMMARY}" + + sync "${BUILD_FNAME}" "${VERSION_FNAME}" "${CRASH_SUMMARY}" # Create the dump compressed pack. LOG_FNAME="steamos-${SN}-${STEAM_ACCOUNT}.${CURRENT_TSTAMP}.zip" LOG_FNAME="${KDUMP_MAIN_FOLDER}/${LOG_FNAME}" + zip -9 -jq "${LOG_FNAME}" "${KDUMP_LOGS_FOLDER}"/* 1>/dev/null 2>&1 - zip -9 -jq "${LOG_FNAME}" ${KDUMP_LOGS_FOLDER}/* 1>/dev/null 2>&1 - sync "${LOG_FNAME}" + sync "${LOG_FNAME}" 2>/dev/null + if [ ! -s "${LOG_FNAME}" ]; then + logger "kdump-steamos: couldn't create the log archive, aborting..." + exit 0 + fi + + + ############################## + # Log submission mechanism # + ############################## + + + # The POST request requires a valid Steam ID. + if [ "${STEAM_ID}" -eq 0 ]; then + logger "kdump-steamos: invalid Steam ID, cannot submit logs" + exit 0 + fi + + # Construct the POST request fields... + REQ_DUMP_SZ="$(stat --printf="%s" "${LOG_FNAME}")" + REQ_PRODUCT="holo" + REQ_BUILD="$(grep "BUILD_ID" "${BUILD_FNAME}" | cut -f2 -d=)" + REQ_VER="$(cat "${VERSION_FNAME}")" + REQ_PLATFORM="linux" + REQ_TIME="${CURRENT_EPOCH}" + STACK_SED_EXPR="/ Call Trace\:/,/ RIP\:/p" + REQ_STACK="$(sed -n "${STACK_SED_EXPR}" "${CRASH_SUMMARY}" | sed "1d")" + REQ_NOTE="$(cat "${CRASH_SUMMARY}")" + + POST_REQ="steamid=${STEAM_ID}&have_dump_file=1&dump_file_size=${REQ_DUMP_SZ}&product=${REQ_PRODUCT}&build=${REQ_BUILD}" + POST_REQ="${POST_REQ}&version=${REQ_VER}&platform=${REQ_PLATFORM}&crash_time=${REQ_TIME}&stack=${REQ_STACK}¬e=${REQ_NOTE}&format=json" + + # Now we can safely delete this folder. rm -rf "${KDUMP_LOGS_FOLDER}" - # TODO: implement a log submission mechanism, in order to send the zip file - # to Valve servers through an API. + # Network validation before log submission + LOOP_CNT=0 + MAX_LOOP=99 + TEST_URL="steampowered.com" + + while [ ${LOOP_CNT} -lt ${MAX_LOOP} ]; do + if ping -i 0.5 -w 2 -c 2 "${TEST_URL}" 1>/dev/null 2>&1; then + break + fi + LOOP_CNT=$((LOOP_CNT + 1)) + sleep 1 + done + + # Bail out in case we have network issues + if [ ${LOOP_CNT} -ge ${MAX_LOOP} ]; then + logger "kdump-steamos: network issue - cannot send logs" + exit 0 + fi + + CURL_ERR="${KDUMP_MAIN_FOLDER}/.curl_err" + START_URL="$(echo "${POST_URL}" | sed 's/ACTION/Start/g')" + FINISH_URL="$(echo "${POST_URL}" | sed 's/ACTION/Finish/g')" + RESPONSE_FILE="${KDUMP_MAIN_FOLDER}/.curl_response" + + if ! curl -X POST -d "${POST_REQ}" "${START_URL}" 1>"${RESPONSE_FILE}" 2>"${CURL_ERR}"; then + logger "kdump-steamos: curl issues - failed in the log submission POST (err=$?)" + #rm -f "${RESPONSE_FILE}" # keep this for now, as debug information + exit 0 + fi + + RESPONSE_PUT_URL="$(jq -r '.response.url' "${RESPONSE_FILE}")" + RESPONSE_GID="$(jq -r '.response.gid' "${RESPONSE_FILE}")" + + # Construct the PUT request based on the POST response + CURL_PUT_HEADERS="${KDUMP_MAIN_FOLDER}/.curl_put_headers" + PUT_HEADERS_LEN=$(jq '.response.headers.pairs | length' "${RESPONSE_FILE}") + + # Validate the response headers; allow a maximum of 20 arguments for now... + if [ "${PUT_HEADERS_LEN}" -le 0 ] || [ "${PUT_HEADERS_LEN}" -gt 20 ]; then + logger "kdump-steamos: unsupported number of response headers (${PUT_HEADERS_LEN}), aborting..." + #rm -f "${RESPONSE_FILE}" # keep this for now, as debug information + exit 0 + fi + + LOOP_CNT=0 + while [ ${LOOP_CNT} -lt "${PUT_HEADERS_LEN}" ]; do + NAME="$(jq -r ".response.headers.pairs[${LOOP_CNT}].name" "${RESPONSE_FILE}")" + VAL="$(jq -r ".response.headers.pairs[${LOOP_CNT}].value" "${RESPONSE_FILE}")" + + echo "${NAME}: ${VAL}" >> "${CURL_PUT_HEADERS}" + LOOP_CNT=$((LOOP_CNT + 1)) + done + + rm -f "${RESPONSE_FILE}" + if ! curl -X PUT --data-binary "@${LOG_FNAME}" -H "@${CURL_PUT_HEADERS}" "${RESPONSE_PUT_URL}" 1>/dev/null 2>"${CURL_ERR}"; then + logger "kdump-steamos: curl issues - failed in the log submission PUT (err=$?)" + #rm -f "${CURL_PUT_HEADERS}" # keep this for now, as debug information + exit 0 + fi + + rm -f "${CURL_PUT_HEADERS}" + if ! curl -X POST -d "gid=${RESPONSE_GID}" "${FINISH_URL}" 1>/dev/null 2>"${CURL_ERR}"; then + logger "kdump-steamos: curl issues - failed in the log finish POST (err=$?)" + exit 0 + fi + + # If we reached this point, the zipped log should have been submitted + # succesfully; save a local copy as well. + # TODO: implement a clean-up routine to just keep up to N logs... + + rm -f "${CURL_ERR}" + SENT_FLD="${KDUMP_MAIN_FOLDER}/sent_logs/" + mkdir -p "${SENT_FLD}" + + mv "${LOG_FNAME}" "${SENT_FLD}" + logger "kdump-steamos: successfully submitted crash log to Valve" fi