ctdb/config/debug-hung-script.sh

   1 #!/bin/sh
   2
   3 # This script only works on Linux.  Please modify (and submit patches)
   4 # for other operating systems.
   5
   6 [ -n "$CTDB_BASE" ] || \
   7     CTDB_BASE=$(d=$(dirname "$0") ; cd -P "$d" ; echo "$PWD")
   8
   9 . "${CTDB_BASE}/functions"
  10
  11 load_script_options
  12
  13 # Testing hook
  14 if [ -n "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE" ] ; then
  15     tmp="${CTDB_DEBUG_HUNG_SCRIPT_LOGFILE}.part"
  16     exec >>"$tmp" 2>&1
  17 fi
  18
  19 (
  20     # No use running several of these in parallel if, say, "releaseip"
  21     # event hangs for multiple IPs.  In that case the output would be
  22     # interleaved in the log and would just be confusing.
  23     flock --wait 2 9 || exit 1
  24
  25     echo "===== Start of hung script debug for PID=\"$1\", event=\"$2\" ====="
  26
  27     echo "pstree -p -a ${1}:"
  28     out=$(pstree -p -a "$1")
  29     echo "$out"
  30
  31     # Check for processes matching a regular expression and print
  32     # stack staces.  This could help confirm that certain processes
  33     # are stuck in certain places such as the cluster filesystem.  The
  34     # regexp must separate items with "|" and must not contain
  35     # parentheses.  The default pattern can be replaced for testing.
  36     default_pat='exportfs|rpcinfo'
  37     pat="${CTDB_DEBUG_HUNG_SCRIPT_STACKPAT:-${default_pat}}"
  38     echo "$out" |
  39     sed -r -n "s@.*-(.*(${pat}).*),([0-9]*).*@\\3 \\1@p" |
  40     while read pid name ; do
  41         trace=$(cat "/proc/${pid}/stack" 2>/dev/null)
  42         # No! Checking the exit code afterwards is actually clearer...
  43         # shellcheck disable=SC2181
  44         if [ $? -eq 0 ] ; then
  45             echo "---- Stack trace of interesting process ${pid}[${name}] ----"
  46             echo "$trace"
  47         fi
  48     done
  49
  50     if [ "$2" != "init" ] ; then
  51         echo "---- ctdb scriptstatus ${2}: ----"
  52         $CTDB scriptstatus "$2"
  53     fi
  54
  55     echo "===== End of hung script debug for PID=\"$1\", event=\"$2\" ====="
  56
  57     if [ -n "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE" ] ; then
  58         mv "$tmp" "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE"
  59     fi
  60
  61 ) 9>"${CTDB_SCRIPT_VARDIR}/debug-hung-script.lock"