ctdb/config/events.d/60.nfs

   1 #!/bin/sh
   2 # script to manage nfs in a clustered environment
   3
   4 [ -n "$CTDB_BASE" ] || \
   5     export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
   6
   7 . $CTDB_BASE/functions
   8
   9 service_name="nfs"
  10 loadconfig
  11 ctdb_setup_service_state_dir
  12
  13 ######################################################################
  14
  15 if [ -z "$CTDB_NFS_CALLOUT" ] ; then
  16     CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
  17 fi
  18 # Always export, for statd callout
  19 export CTDB_NFS_CALLOUT
  20
  21 nfs_callout_cache="${service_state_dir}/nfs_callout_cache"
  22 nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
  23 nfs_callout_cache_ops="${nfs_callout_cache}/ops"
  24
  25 nfs_callout_register ()
  26 {
  27     mkdir -p "$nfs_callout_cache_ops"
  28     rm -f "$nfs_callout_cache_ops"/*
  29
  30     echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"
  31
  32     _t=$(eval "$CTDB_NFS_CALLOUT" "register")
  33     if [ -n "$_t" ] ; then
  34         echo "$_t" |
  35             while IFS="" read _op ; do
  36                 touch "${nfs_callout_cache_ops}/${_op}"
  37             done
  38     else
  39         touch "${nfs_callout_cache_ops}/ALL"
  40     fi
  41 }
  42
  43 nfs_callout ()
  44 {
  45     # Re-run registration if $CTDB_NFS_CALLOUT has changed
  46     _prev=""
  47     if [ -r "$nfs_callout_cache_callout" ] ; then
  48         read _prev <"$nfs_callout_cache_callout"
  49     fi
  50     if [ "$CTDB_NFS_CALLOUT" != "$_prev" ] ; then
  51         nfs_callout_register
  52     fi
  53
  54     # Run the operation if it is registered...
  55     if [ -e "${nfs_callout_cache_ops}/${1}" ] || \
  56            [ -e "${nfs_callout_cache_ops}/ALL" ]; then
  57         eval "$CTDB_NFS_CALLOUT" "$@"
  58     fi
  59 }
  60
  61 service_reconfigure ()
  62 {
  63     # Restart lock manager, notify clients
  64     if [ -x "${CTDB_BASE}/statd-callout" ] ; then
  65         "${CTDB_BASE}/statd-callout" notify &
  66     fi >/dev/null 2>&1
  67 }
  68
  69 ######################################################################
  70
  71 ######################################################
  72 # Check the health of NFS services
  73 #
  74 # Use .check files in given directory.
  75 # Default is "${CTDB_BASE}/nfs-checks.d/"
  76 ######################################################
  77 nfs_check_services ()
  78 {
  79     _dir="${1:-${CTDB_NFS_CHECKS_DIR:-${CTDB_BASE}/nfs-checks.d}}"
  80
  81     # Files must end with .check - avoids editor backups, RPM fu, ...
  82     for _f in "$_dir"/[0-9][0-9].*.check ; do
  83         _t="${_f%.check}"
  84         _progname="${_t##*/[0-9][0-9].}"
  85
  86         nfs_check_service "$_progname" <"$_f"
  87     done
  88 }
  89
  90 ######################################################
  91 # Check the health of an NFS service
  92 #
  93 # $1 - progname, passed to rpcinfo (looked up in /etc/rpc)
  94 #
  95 # Reads variables from stdin
  96 #
  97 # Variables are:
  98 #
  99 # * family             - "tcp" or "udp" or space separated list
 100 #                        default: tcp, not used with "service_check_cmd"
 101 # * version            - optional, RPC service version number
 102 #                        default is to omit to check for any version,
 103 #                        not used with "service_check_cmd"
 104 # * unhealthy_after    - number of check fails before unhealthy
 105 #                        default: 1
 106 # * restart_every      - number of check fails before restart
 107 #                        default: 0, meaning no restart
 108 # * service_stop_cmd   - command to stop service
 109 #                        default: no default, must be provided if
 110 #                                 restart_every > 0
 111 # * service_start_cmd  - command to start service
 112 #                        default: no default, must be provided if
 113 #                                 restart_every > 0
 114 # * service_check_cmd  - command to check health of service
 115 #                        default is to check RPC service using rpcinfo
 116 # * service_debug_cmd  - command to debug a service after trying to stop it;
 117 #                        for example, it can be useful to print stack
 118 #                        traces of threads that have not exited, since
 119 #                        they may be stuck doing I/O;
 120 #                        no default, see also function program_stack_traces()
 121 #
 122 # Quoting in values is not preserved
 123 #
 124 ######################################################
 125 nfs_check_service ()
 126 {
 127     _progname="$1"
 128
 129     (
 130         # Subshell to restrict scope variables...
 131
 132         # Defaults
 133         family="tcp"
 134         version=""
 135         unhealthy_after=1
 136         restart_every=0
 137         service_stop_cmd=""
 138         service_start_cmd=""
 139         service_check_cmd=""
 140         service_debug_cmd=""
 141
 142         # Eval line-by-line.  Expands variable references in values.
 143         # Also allows variable name checking, which seems useful.
 144         while read _line ; do
 145             case "$_line" in
 146                 \#*|"") : ;; # Ignore comments, blank lines
 147
 148                 family=*|version=*|\
 149                 unhealthy_after=*|restart_every=*|\
 150                 service_stop_cmd=*|service_start_cmd=*|\
 151                 service_check_cmd=*|service_debug_cmd=*)
 152
 153                     eval "$_line"
 154                     ;;
 155                 *)
 156                     echo "ERROR: Unknown variable for ${_progname}: ${_line}"
 157                     exit 1
 158             esac
 159         done
 160
 161         _service_name="nfs_${_progname}"
 162
 163         _ok=false
 164         if [ -n "$service_check_cmd" ] ; then
 165             # Using eval means variables can contain semicolon separated commands
 166             if eval "$service_check_cmd" ; then
 167                 _ok=true
 168             fi
 169         else
 170             if nfs_check_rpcinfo \
 171                    "$_progname" "$version" "$family" >/dev/null ; then
 172                 _ok=true
 173             fi
 174         fi
 175
 176         if $_ok ; then
 177             if [ $unhealthy_after -ne 1 -o $restart_every -ne 0 ] ; then
 178                 ctdb_counter_init "$_service_name"
 179             fi
 180             exit 0
 181         fi
 182
 183         ctdb_counter_incr "$_service_name"
 184         _failcount=$(ctdb_counter_get "$_service_name")
 185
 186         _unhealthy=false
 187         if [ $unhealthy_after -gt 0 ] ; then
 188             if [ $_failcount -ge $unhealthy_after ] ; then
 189                 _unhealthy=true
 190                 echo "ERROR: $ctdb_check_rpc_out"
 191             fi
 192         fi
 193
 194         if [ $restart_every -gt 0 ] ; then
 195             if [ $(($_failcount % $restart_every)) -eq 0 ] ; then
 196                 if ! $_unhealthy ; then
 197                     echo "WARNING: $ctdb_check_rpc_out"
 198                 fi
 199                 nfs_restart_service
 200             fi
 201         fi
 202
 203         if $_unhealthy ; then
 204             exit 1
 205         fi
 206
 207         return 0
 208     ) || exit 1
 209 }
 210
 211 # Uses: stop_service, start_service, debug_stuck_threads
 212 nfs_restart_service ()
 213 {
 214     if [ -z "$service_stop_cmd" -o -z "$service_start_cmd" ] ; then
 215         die "ERROR: Can not restart service \"${_progname}\" without corresponding service_start_cmd/service_stop_cmd settings"
 216     fi
 217
 218     echo "Trying to restart service \"${_progname}\"..."
 219     # Using eval means variables can contain semicolon separated commands
 220     eval "$service_stop_cmd"
 221     if [ -n "$service_debug_cmd" ] ; then
 222         eval "$service_debug_cmd"
 223     fi
 224     background_with_logging eval "$service_start_cmd"
 225 }
 226
 227 ######################################################
 228 # Check an RPC service with rpcinfo
 229 ######################################################
 230 ctdb_check_rpc ()
 231 {
 232     _progname="$1"        # passed to rpcinfo (looked up in /etc/rpc)
 233     _version="$2"         # optional, not passed if empty/unset
 234     _family="${3:-tcp}"   # optional, default is "tcp"
 235
 236     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 237
 238     if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost \
 239                                       $_progname $_version 2>&1) ; then
 240         ctdb_check_rpc_out="$_progname failed RPC check:
 241 $ctdb_check_rpc_out"
 242         echo "$ctdb_check_rpc_out"
 243         return 1
 244     fi
 245 }
 246
 247 nfs_check_rpcinfo ()
 248 {
 249     _progname="$1"        # passed to rpcinfo (looked up in /etc/rpc)
 250     _versions="$2"        # optional, space separated, not passed if empty/unset
 251     _families="${3:-tcp}" # optional, space separated, default is "tcp"
 252
 253     for _family in $_families ; do
 254         if [ -n "$_versions" ] ; then
 255             for _version in $_versions ; do
 256                 ctdb_check_rpc $_progname $_version $_family || return $?
 257             done
 258         else
 259             ctdb_check_rpc $_progname "" $_family || return $?
 260         fi
 261     done
 262 }
 263
 264 ##################################################################
 265 # use statd-callout to update NFS lock info
 266 ##################################################################
 267 nfs_update_lock_info ()
 268 {
 269     if [ -x "$CTDB_BASE/statd-callout" ] ; then
 270         "$CTDB_BASE/statd-callout" update
 271     fi
 272 }
 273
 274 ######################################################################
 275
 276 ctdb_start_stop_service
 277
 278 is_ctdb_managed_service || exit 0
 279
 280 ctdb_service_check_reconfigure
 281
 282 case "$1" in
 283     startup)
 284         nfs_callout "$@"
 285         ;;
 286
 287     shutdown)
 288          nfs_callout "$@"
 289         ;;
 290
 291     takeip)
 292         nfs_callout "$@"
 293         ctdb_service_set_reconfigure
 294         ;;
 295
 296     releaseip)
 297         nfs_callout "$@"
 298         ctdb_service_set_reconfigure
 299         ;;
 300
 301     monitor)
 302         nfs_callout "monitor-pre" || exit $?
 303
 304         # Check that directories for shares actually exist
 305         if [ "$CTDB_NFS_SKIP_SHARE_CHECK" != "yes" ] ; then
 306             nfs_callout "monitor-list-shares" | ctdb_check_directories || \
 307                 exit $?
 308         fi
 309
 310         update_tickles 2049
 311         nfs_update_lock_info
 312
 313         nfs_check_services
 314
 315         nfs_callout "monitor-post" || exit $?
 316         ;;
 317
 318     *)
 319         ctdb_standard_event_handler "$@"
 320         ;;
 321 esac
 322
 323 exit 0