ctdb/config/debug-hung-script.sh

   1 #!/bin/sh
   2
   3 [ -n "$CTDB_BASE" ] || \
   4     export CTDB_BASE=$(cd -P $(dirname "$0") ; echo "$PWD")
   5
   6 . "$CTDB_BASE/functions"
   7
   8 loadconfig ctdb
   9
  10 # Testing hook
  11 if [ -n "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE" ] ; then
  12     tmp="${CTDB_DEBUG_HUNG_SCRIPT_LOGFILE}.part"
  13     exec >>"$tmp" 2>&1
  14 fi
  15
  16 (
  17     # No use running several of these in parallel if, say, "releaseip"
  18     # event hangs for multiple IPs.  In that case the output would be
  19     # interleaved in the log and would just be confusing.
  20     flock --wait 2 9 || exit 1
  21
  22     echo "===== Start of hung script debug for PID=\"$1\", event=\"$2\" ====="
  23
  24     echo "pstree -p -a ${1}:"
  25     out=$(pstree -p -a $1)
  26     echo "$out"
  27
  28     # Check for processes matching a regular expression and print
  29     # stack staces.  This could help confirm that certain processes
  30     # are stuck in certain places such as the cluster filesystem.  The
  31     # regexp should separate items with "\|" and should not contain
  32     # parentheses.  The default pattern can be replaced for testing.
  33     default_pat='exportfs\|rpcinfo'
  34     pat="${CTDB_DEBUG_HUNG_SCRIPT_STACKPAT:-${default_pat}}"
  35     echo "$out" |
  36     sed -n "s@.*-\(.*${pat}.*\),\([0-9]*\).*@\2 \1@p" |
  37     while read pid name ; do
  38         trace=$(cat "/proc/${pid}/stack" 2>/dev/null)
  39         if [ $? -eq 0 ] ; then
  40             echo "---- Stack trace of interesting process ${pid}[${name}] ----"
  41             echo "$trace"
  42         fi
  43     done
  44
  45     if [ "$2" != "init" ] ; then
  46         echo "---- ctdb scriptstatus ${2}: ----"
  47         ctdb scriptstatus "$2"
  48     fi
  49
  50     echo "===== End of hung script debug for PID=\"$1\", event=\"$2\" ====="
  51
  52     if [ -n "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE" ] ; then
  53         mv "$tmp" "$CTDB_DEBUG_HUNG_SCRIPT_LOGFILE"
  54     fi
  55
  56 ) 9>"${CTDB_VARDIR}/debug-hung-script.lock"