ctdb/config/functions

   1 # Hey Emacs, this is a -*- shell-script -*- !!!
   2
   3 # utility functions for ctdb event scripts
   4
   5 [ -z "$CTDB_VARDIR" ] && {
   6     if [ -d "/var/lib/ctdb" ] ; then
   7         export CTDB_VARDIR="/var/lib/ctdb"
   8     else
   9         export CTDB_VARDIR="/var/ctdb"
  10     fi
  11 }
  12 [ -z "$CTDB_ETCDIR" ] && {
  13     export CTDB_ETCDIR="/etc"
  14 }
  15
  16 #######################################
  17 # pull in a system config file, if any
  18 _loadconfig() {
  19
  20     if [ -z "$1" ] ; then
  21         foo="${service_config:-${service_name}}"
  22         if [ -n "$foo" ] ; then
  23             loadconfig "$foo"
  24             return
  25         fi
  26     fi
  27
  28     if [ "$1" != "ctdb" ] ; then
  29         loadconfig "ctdb"
  30     fi
  31
  32     if [ -z "$1" ] ; then
  33         return
  34     fi
  35
  36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
  37         . $CTDB_ETCDIR/sysconfig/$1
  38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
  39         . $CTDB_ETCDIR/default/$1
  40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
  41         . $CTDB_BASE/sysconfig/$1
  42     fi
  43
  44     if [ "$1" = "ctdb" ] ; then
  45         _config="${CTDB_BASE}/ctdbd.conf"
  46         if [ -r "$_config" ] ; then
  47             . "$_config"
  48         fi
  49     fi
  50 }
  51
  52 loadconfig () {
  53     _loadconfig "$@"
  54 }
  55
  56 ##############################################################
  57
  58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
  59 # configuration file.
  60 debug ()
  61 {
  62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
  63         # If there are arguments then echo them.  Otherwise expect to
  64         # use stdin, which allows us to pass lots of debug using a
  65         # here document.
  66         if [ -n "$1" ] ; then
  67             echo "DEBUG: $*"
  68         else
  69             sed -e 's@^@DEBUG: @'
  70         fi
  71     else
  72         if [ -z "$1" ] ; then
  73             cat >/dev/null
  74         fi
  75     fi
  76 }
  77
  78 die ()
  79 {
  80     _msg="$1"
  81     _rc="${2:-1}"
  82
  83     echo "$_msg"
  84     exit $_rc
  85 }
  86
  87 # Log given message or stdin to either syslog or a CTDB log file
  88 # $1 is the tag passed to logger if syslog is in use.
  89 script_log ()
  90 {
  91     _tag="$1" ; shift
  92
  93     case "$CTDB_LOGGING" in
  94         file:*|"")
  95             if [ -n "$CTDB_LOGGING" ] ; then
  96                 _file="${CTDB_LOGGING#file:}"
  97             else
  98                 _file="/var/log/log.ctdb"
  99             fi
 100             {
 101                 if [ -n "$*" ] ; then
 102                     echo "$*"
 103                 else
 104                     cat
 105                 fi
 106             } >>"$_file"
 107             ;;
 108         *)
 109             # Handle all syslog:* variants here too.  There's no tool to do
 110             # the lossy things, so just use logger.
 111             logger -t "ctdbd: ${_tag}" $*
 112             ;;
 113     esac
 114 }
 115
 116 # When things are run in the background in an eventscript then logging
 117 # output might get lost.  This is the "solution".  :-)
 118 background_with_logging ()
 119 {
 120     (
 121         "$@" 2>&1 </dev/null |
 122         script_log "${script_name}&"
 123     )&
 124
 125     return 0
 126 }
 127
 128 ##############################################################
 129 # check number of args for different events
 130 ctdb_check_args ()
 131 {
 132     case "$1" in
 133         takeip|releaseip)
 134             if [ $# != 4 ]; then
 135                 echo "ERROR: must supply interface, IP and maskbits"
 136                 exit 1
 137             fi
 138             ;;
 139         updateip)
 140             if [ $# != 5 ]; then
 141                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
 142                 exit 1
 143             fi
 144             ;;
 145     esac
 146 }
 147
 148 ##############################################################
 149 # determine on what type of system (init style) we are running
 150 detect_init_style()
 151 {
 152     # only do detection if not already set:
 153     [ -z "$CTDB_INIT_STYLE" ] || return
 154
 155     if [ -x /sbin/startproc ]; then
 156         CTDB_INIT_STYLE="suse"
 157     elif [ -x /sbin/start-stop-daemon ]; then
 158         CTDB_INIT_STYLE="debian"
 159     else
 160         CTDB_INIT_STYLE="redhat"
 161     fi
 162 }
 163
 164 ######################################################
 165 # simulate /sbin/service on platforms that don't have it
 166 # _service() makes it easier to hook the service() function for
 167 # testing.
 168 _service ()
 169 {
 170   _service_name="$1"
 171   _op="$2"
 172
 173   # do nothing, when no service was specified
 174   [ -z "$_service_name" ] && return
 175
 176   if [ -x /sbin/service ]; then
 177       $_nice /sbin/service "$_service_name" "$_op"
 178   elif [ -x /usr/sbin/service ]; then
 179       $_nice /usr/sbin/service "$_service_name" "$_op"
 180   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
 181       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
 182   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
 183       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
 184   fi
 185 }
 186
 187 service()
 188 {
 189     _nice=""
 190     _service "$@"
 191 }
 192
 193 ######################################################
 194 # simulate /sbin/service (niced) on platforms that don't have it
 195 nice_service()
 196 {
 197     _nice="nice"
 198     _service "$@"
 199 }
 200
 201 ######################################################
 202 # Cached retrieval of PNN from local node.  This never changes so why
 203 # open a client connection to the server each time this is needed?
 204 # This sets $pnn - this avoid an unnecessary subprocess.
 205 ctdb_get_pnn ()
 206 {
 207     _pnn_file="$CTDB_VARDIR/state/my-pnn"
 208     if [ ! -f "$_pnn_file" ] ; then
 209         ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
 210     fi
 211
 212     read pnn <"$_pnn_file"
 213 }
 214
 215 ######################################################
 216 # wrapper around /proc/ settings to allow them to be hooked
 217 # for testing
 218 # 1st arg is relative path under /proc/, 2nd arg is value to set
 219 set_proc ()
 220 {
 221     echo "$2" >"/proc/$1"
 222 }
 223
 224 set_proc_maybe ()
 225 {
 226     if [ -w "/proc/$1" ] ; then
 227         set_proc "$1" "$2"
 228     fi
 229 }
 230
 231 ######################################################
 232 # wrapper around getting file contents from /proc/ to allow
 233 # this to be hooked for testing
 234 # 1st arg is relative path under /proc/
 235 get_proc ()
 236 {
 237     cat "/proc/$1"
 238 }
 239
 240 ######################################################
 241 # Print up to $_max kernel stack traces for processes named $_program
 242 program_stack_traces ()
 243 {
 244     _prog="$1"
 245     _max="${2:-1}"
 246
 247     _count=1
 248     for _pid in $(pidof "$_prog") ; do
 249         [ $_count -le $_max ] || break
 250
 251         # Do this first to avoid racing with process exit
 252         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
 253         if [ -n "$_stack" ] ; then
 254             echo "Stack trace for ${_prog}[${_pid}]:"
 255             echo "$_stack"
 256             _count=$(($_count + 1))
 257         fi
 258     done
 259 }
 260
 261 ######################################################
 262 # Check that an RPC service is healthy -
 263 # this includes allowing a certain number of failures
 264 # before marking the NFS service unhealthy.
 265 #
 266 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
 267 #
 268 # each triple is a set of 3 arguments: an operator, a
 269 # fail count limit and an action string.
 270 #
 271 # For example:
 272 #
 273 #       nfs_check_rpc_service "lockd" \
 274 #           -ge 15 "verbose restart unhealthy" \
 275 #           -eq 10 "restart:bs"
 276 #
 277 # says that if lockd is down for 15 iterations then do
 278 # a verbose restart of lockd and mark the node unhealthy.
 279 # Before this, after 10 iterations of failure, the
 280 # service is restarted silently in the background.
 281 # Order is important: the number of failures need to be
 282 # specified in reverse order because processing stops
 283 # after the first condition that is true.
 284 ######################################################
 285 nfs_check_rpc_service ()
 286 {
 287     _prog_name="$1" ; shift
 288
 289     if _nfs_check_rpc_common "$_prog_name" ; then
 290         return
 291     fi
 292
 293     while [ -n "$3" ] ; do
 294         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
 295             break
 296         fi
 297         shift 3
 298     done
 299 }
 300
 301 # The new way of doing things...
 302 nfs_check_rpc_services ()
 303 {
 304     # Files must end with .check - avoids editor backups, RPM fu, ...
 305     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
 306         _t="${_f%.check}"
 307         _prog_name="${_t##*/[0-9][0-9].}"
 308
 309         # If $_prog_name contains '@' then the bit after it is the
 310         # address family.
 311         _family="${_prog_name#*@}"
 312         if [ "$_family" = "$_prog_name" ] ; then
 313             _family=""
 314         else
 315             _prog_name="${_prog_name%@*}"
 316         fi
 317
 318         if _nfs_check_rpc_common "$_prog_name" "$_family" ; then
 319             # This RPC service is up, check next service...
 320             continue
 321         fi
 322
 323         # Check each line in the file in turn until one of the limit
 324         # checks is hit...
 325         while read _cmp _lim _rest ; do
 326             # Skip comments
 327             case "$_cmp" in
 328                 \#*) continue ;;
 329             esac
 330
 331             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
 332                 # Limit was hit on this line, no further checking...
 333                 break
 334             fi
 335         done <"$_f"
 336     done
 337 }
 338
 339 _nfs_check_rpc_common ()
 340 {
 341     _prog_name="$1"
 342     _family="$2"
 343
 344     # Some platforms don't have separate programs for all services.
 345     case "$_prog_name" in
 346         statd)
 347             type "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
 348     esac
 349
 350     case "$_prog_name" in
 351         nfsd)
 352             _rpc_prog=nfs
 353             _version=3
 354             ;;
 355         mountd)
 356             _rpc_prog=mountd
 357             _version=1
 358             ;;
 359         rquotad)
 360             _rpc_prog=rquotad
 361             _version=1
 362             ;;
 363         lockd)
 364             _rpc_prog=nlockmgr
 365             _version=4
 366             ;;
 367         statd)
 368             _rpc_prog=status
 369             _version=1
 370             ;;
 371         *)
 372             echo "Internal error: unknown RPC program \"$_prog_name\"."
 373             exit 1
 374     esac
 375
 376     _service_name="nfs_${_prog_name}${_family:+_}${_family}"
 377
 378     if ctdb_check_rpc "$_rpc_prog" "$_version" "$_family" >/dev/null ; then
 379         ctdb_counter_init "$_service_name"
 380         return 0
 381     fi
 382
 383     ctdb_counter_incr "$_service_name"
 384
 385     return 1
 386 }
 387
 388 _nfs_check_rpc_action ()
 389 {
 390     _cmp="$1"
 391     _limit="$2"
 392     _actions="$3"
 393
 394     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
 395         return 1
 396     fi
 397
 398     for _action in $_actions ; do
 399         case "$_action" in
 400             verbose)
 401                 echo "ERROR: $ctdb_check_rpc_out"
 402                 ;;
 403             restart)
 404                 _nfs_restart_rpc_service "$_prog_name"
 405                 ;;
 406             restart:b)
 407                 _nfs_restart_rpc_service "$_prog_name" true
 408                 ;;
 409             unhealthy)
 410                 exit 1
 411                 ;;
 412             *)
 413                 echo "Internal error: unknown action \"$_action\"."
 414                 exit 1
 415         esac
 416     done
 417
 418     return 0
 419 }
 420
 421 _nfs_restart_rpc_service ()
 422 {
 423     _prog_name="$1"
 424     _background="${2:-false}"
 425
 426     if $_background ; then
 427         _maybe_background="background_with_logging"
 428     else
 429         _maybe_background=""
 430     fi
 431
 432     _p="rpc.${_prog_name}"
 433
 434     case "$_prog_name" in
 435         nfsd)
 436             echo "Trying to restart NFS service"
 437             $_maybe_background startstop_nfs restart
 438             ;;
 439         mountd)
 440             echo "Trying to restart $_prog_name [${_p}]"
 441             killall -q -9 "$_p"
 442             nfs_dump_some_threads "$_p"
 443             $_maybe_background $_p $RPCMOUNTDOPTS \
 444                                ${MOUNTD_PORT:+-p} $MOUNTD_PORT
 445             ;;
 446         rquotad)
 447             echo "Trying to restart $_prog_name [${_p}]"
 448             killall -q -9 "$_p"
 449             nfs_dump_some_threads "$_p"
 450             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
 451             ;;
 452         lockd)
 453             echo "Trying to restart lock manager service"
 454             $_maybe_background startstop_nfslock restart
 455             ;;
 456         statd)
 457             echo "Trying to restart $_prog_name [${_p}]"
 458             killall -q -9 "$_p"
 459             nfs_dump_some_threads "$_p"
 460             $_maybe_background $_p \
 461                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
 462                 ${STATD_PORT:+-p} $STATD_PORT \
 463                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
 464             ;;
 465         *)
 466             echo "Internal error: unknown RPC program \"$_prog_name\"."
 467             exit 1
 468     esac
 469 }
 470
 471 ######################################################
 472 # Check an RPC service with rpcinfo
 473 ######################################################
 474 ctdb_check_rpc ()
 475 {
 476     _progname="$1"        # passed to rpcinfo (looked up in /etc/rpc)
 477     _version="$2"         # optional, not passed if empty/unset
 478     _family="${3:-tcp}"   # optional, default is "tcp"
 479
 480     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
 481
 482     if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost \
 483                                       $_progname $_version 2>&1) ; then
 484         ctdb_check_rpc_out="$_progname failed RPC check:
 485 $ctdb_check_rpc_out"
 486         echo "$ctdb_check_rpc_out"
 487         return 1
 488     fi
 489 }
 490
 491 ######################################################
 492 # Ensure $service_name is set
 493 assert_service_name ()
 494 {
 495     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
 496 }
 497
 498 ######################################################
 499 # check a set of directories is available
 500 # return 1 on a missing directory
 501 # directories are read from stdin
 502 ######################################################
 503 ctdb_check_directories_probe()
 504 {
 505     while IFS="" read d ; do
 506         case "$d" in
 507             *%*)
 508                 continue
 509                 ;;
 510             *)
 511                 [ -d "${d}/." ] || return 1
 512         esac
 513     done
 514 }
 515
 516 ######################################################
 517 # check a set of directories is available
 518 # directories are read from stdin
 519 ######################################################
 520 ctdb_check_directories()
 521 {
 522     ctdb_check_directories_probe || {
 523         echo "ERROR: $service_name directory \"$d\" not available"
 524         exit 1
 525     }
 526 }
 527
 528 ######################################################
 529 # check a set of tcp ports
 530 # usage: ctdb_check_tcp_ports <ports...>
 531 ######################################################
 532
 533 # This flag file is created when a service is initially started.  It
 534 # is deleted the first time TCP port checks for that service succeed.
 535 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
 536 # message if a port check fails.
 537 _ctdb_check_tcp_common ()
 538 {
 539     assert_service_name
 540     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
 541 }
 542
 543 ctdb_check_tcp_init ()
 544 {
 545     _ctdb_check_tcp_common
 546     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
 547     touch "$_ctdb_service_started_file"
 548 }
 549
 550 # Check whether something is listening on all of the given TCP ports
 551 # using the "ctdb checktcpport" command.
 552 ctdb_check_tcp_ports()
 553 {
 554     if [ -z "$1" ] ; then
 555         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
 556         exit 1
 557     fi
 558
 559     for _p ; do  # process each function argument (port)
 560         _cmd="ctdb checktcpport $_p"
 561         _out=$($_cmd 2>&1)
 562         _ret=$?
 563         case "$_ret" in
 564             0)
 565                 _ctdb_check_tcp_common
 566                 if [ ! -f "$_ctdb_service_started_file" ] ; then
 567                     echo "ERROR: $service_name tcp port $_p is not responding"
 568                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
 569                 else
 570                     echo "INFO: $service_name tcp port $_p is not responding"
 571                 fi
 572
 573                 return 1
 574                 ;;
 575             98)
 576                 # Couldn't bind, something already listening, next port...
 577                 continue
 578                 ;;
 579             *)
 580                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
 581                 debug <<EOF
 582 ctdb checktcpport (exited with $_ret) with output:
 583 $_out"
 584 EOF
 585                 return $_ret
 586         esac
 587     done
 588
 589     # All ports listening
 590     _ctdb_check_tcp_common
 591     rm -f "$_ctdb_service_started_file"
 592     return 0
 593 }
 594
 595 ######################################################
 596 # check a unix socket
 597 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
 598 ######################################################
 599 ctdb_check_unix_socket() {
 600     socket_path="$1"
 601     [ -z "$socket_path" ] && return
 602
 603     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
 604         echo "ERROR: $service_name socket $socket_path not found"
 605         return 1
 606     fi
 607 }
 608
 609 ######################################################
 610 # check a command returns zero status
 611 # usage: ctdb_check_command <command>
 612 ######################################################
 613 ctdb_check_command ()
 614 {
 615     _out=$("$@" 2>&1) || {
 616         echo "ERROR: $* returned error"
 617         echo "$_out" | debug
 618         exit 1
 619     }
 620 }
 621
 622 ################################################
 623 # kill off any TCP connections with the given IP
 624 ################################################
 625 kill_tcp_connections ()
 626 {
 627     _ip="$1"
 628
 629     _oneway=false
 630     if [ "$2" = "oneway" ] ; then
 631         _oneway=true
 632     fi
 633
 634     get_tcp_connections_for_ip "$_ip" | {
 635         _killcount=0
 636         _connections=""
 637         _nl="
 638 "
 639         while read _dst _src; do
 640             _destport="${_dst##*:}"
 641             __oneway=$_oneway
 642             case $_destport in
 643                 # we only do one-way killtcp for CIFS
 644                 139|445) __oneway=true ;;
 645             esac
 646
 647             echo "Killing TCP connection $_src $_dst"
 648             _connections="${_connections}${_nl}${_src} ${_dst}"
 649             if ! $__oneway ; then
 650                 _connections="${_connections}${_nl}${_dst} ${_src}"
 651             fi
 652
 653             _killcount=$(($_killcount + 1))
 654         done
 655
 656         if [ $_killcount -eq 0 ] ; then
 657             return
 658         fi
 659
 660         echo "$_connections" | ctdb killtcp || {
 661             echo "Failed to send killtcp control"
 662             return
 663         }
 664
 665         _count=0
 666         while : ; do
 667             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
 668
 669             if [ $_remaining -eq 0 ] ; then
 670                 echo "Killed $_killcount TCP connections to released IP $_ip"
 671                 return
 672             fi
 673
 674             _count=$(($_count + 1))
 675             if [ $_count -gt 3 ] ; then
 676                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
 677                 return
 678             fi
 679
 680             echo "Waiting for $_remaining connections to be killed for IP $_ip"
 681             sleep 1
 682         done
 683     }
 684 }
 685
 686 ##################################################################
 687 # kill off the local end for any TCP connections with the given IP
 688 ##################################################################
 689 kill_tcp_connections_local_only ()
 690 {
 691     kill_tcp_connections "$1" "oneway"
 692 }
 693
 694 ##################################################################
 695 # tickle any TCP connections with the given IP
 696 ##################################################################
 697 tickle_tcp_connections ()
 698 {
 699     _ip="$1"
 700
 701     get_tcp_connections_for_ip "$_ip" |
 702     {
 703         _failed=false
 704
 705         while read dest src; do
 706             echo "Tickle TCP connection $src $dest"
 707             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
 708             echo "Tickle TCP connection $dest $src"
 709             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
 710         done
 711
 712         if $_failed ; then
 713             echo "Failed to send tickle control"
 714         fi
 715     }
 716 }
 717
 718 get_tcp_connections_for_ip ()
 719 {
 720     _ip="$1"
 721
 722     netstat -tn | awk -v ip=$_ip \
 723         'index($1, "tcp") == 1 && \
 724          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
 725          && $6 == "ESTABLISHED" \
 726          {print $4" "$5}'
 727 }
 728
 729 ##################################################################
 730 # use statd-callout to update NFS lock info
 731 ##################################################################
 732 nfs_update_lock_info ()
 733 {
 734     if [ -x "$CTDB_BASE/statd-callout" ] ; then
 735         "$CTDB_BASE/statd-callout" update
 736     fi
 737 }
 738
 739 ########################################################
 740 # start/stop the Ganesha nfs service
 741 ########################################################
 742 startstop_ganesha()
 743 {
 744     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
 745     case "$1" in
 746         start)
 747             service "$_service_name" start
 748             ;;
 749         stop)
 750             service "$_service_name" stop
 751             ;;
 752         restart)
 753             service "$_service_name" stop
 754             nfs_dump_some_threads "rpc.statd"
 755             service "$_service_name" start
 756             ;;
 757     esac
 758 }
 759
 760 ########################################################
 761 # start/stop the nfs service on different platforms
 762 ########################################################
 763 startstop_nfs() {
 764         PLATFORM="unknown"
 765         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 766                 PLATFORM="sles"
 767         }
 768         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 769             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 770                 PLATFORM="rhel"
 771         }
 772
 773         case $PLATFORM in
 774         sles)
 775                 case $1 in
 776                 start)
 777                         service nfsserver start
 778                         ;;
 779                 stop)
 780                         service nfsserver stop > /dev/null 2>&1
 781                         ;;
 782                 restart)
 783                         set_proc "fs/nfsd/threads" 0
 784                         service nfsserver stop > /dev/null 2>&1
 785                         pkill -9 nfsd
 786                         nfs_dump_some_threads
 787                         service nfsserver start
 788                         ;;
 789                 esac
 790                 ;;
 791         rhel)
 792                 case $1 in
 793                 start)
 794                         service nfslock start
 795                         service nfs start
 796                         ;;
 797                 stop)
 798                         service nfs stop
 799                         service nfslock stop
 800                         ;;
 801                 restart)
 802                         set_proc "fs/nfsd/threads" 0
 803                         service nfs stop > /dev/null 2>&1
 804                         service nfslock stop > /dev/null 2>&1
 805                         pkill -9 nfsd
 806                         nfs_dump_some_threads
 807                         service nfslock start
 808                         service nfs start
 809                         ;;
 810                 esac
 811                 ;;
 812         *)
 813                 echo "Unknown platform. NFS is not supported with ctdb"
 814                 exit 1
 815                 ;;
 816         esac
 817 }
 818
 819 # Dump up to the configured number of nfsd thread backtraces.
 820 nfs_dump_some_threads ()
 821 {
 822     _prog="${1:-nfsd}"
 823
 824     _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
 825     [ $_num -gt 0 ] || return 0
 826
 827     program_stack_traces "$_prog" $_num
 828 }
 829
 830 ########################################################
 831 # start/stop the nfs lockmanager service on different platforms
 832 ########################################################
 833 startstop_nfslock() {
 834         PLATFORM="unknown"
 835         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
 836                 PLATFORM="sles"
 837         }
 838         [ -x $CTDB_ETCDIR/init.d/nfslock -o \
 839             -r /usr/lib/systemd/system/nfs-lock.service ] && {
 840                 PLATFORM="rhel"
 841         }
 842
 843         case $PLATFORM in
 844         sles)
 845                 # for sles there is no service for lockmanager
 846                 # so we instead just shutdown/restart nfs
 847                 case $1 in
 848                 start)
 849                         service nfsserver start
 850                         ;;
 851                 stop)
 852                         service nfsserver stop > /dev/null 2>&1
 853                         ;;
 854                 restart)
 855                         service nfsserver stop > /dev/null 2>&1
 856                         service nfsserver start
 857                         ;;
 858                 esac
 859                 ;;
 860         rhel)
 861                 case $1 in
 862                 start)
 863                         service nfslock start
 864                         ;;
 865                 stop)
 866                         service nfslock stop > /dev/null 2>&1
 867                         ;;
 868                 restart)
 869                         service nfslock stop > /dev/null 2>&1
 870                         service nfslock start
 871                         ;;
 872                 esac
 873                 ;;
 874         *)
 875                 echo "Unknown platform. NFS locking is not supported with ctdb"
 876                 exit 1
 877                 ;;
 878         esac
 879 }
 880
 881 ########################################################
 882
 883 add_ip_to_iface ()
 884 {
 885     _iface=$1
 886     _ip=$2
 887     _maskbits=$3
 888
 889     # Ensure interface is up
 890     ip link set "$_iface" up || \
 891         die "Failed to bringup interface $_iface"
 892
 893     # Only need to define broadcast for IPv4
 894     case "$ip" in
 895         *:*) _bcast=""      ;;
 896         *)   _bcast="brd +" ;;
 897     esac
 898
 899     ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
 900         echo "Failed to add $_ip/$_maskbits on dev $_iface"
 901         return 1
 902     }
 903
 904     # Wait 5 seconds for IPv6 addresses to stop being tentative...
 905     if [ -z "$_bcast" ] ; then
 906         for _x in $(seq 1 10) ; do
 907             ip addr show to "${_ip}/128" | grep -q "tentative" || break
 908             sleep 0.5
 909         done
 910
 911         # If the address was a duplicate then it won't be on the
 912         # interface so flag an error.
 913         _t=$(ip addr show to "${_ip}/128")
 914         case "$_t" in
 915             "")
 916                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 917                 return 1
 918                 ;;
 919             *tentative*|*dadfailed*)
 920                 echo "Failed to add $_ip/$_maskbits on dev $_iface"
 921                 ip addr del "$_ip/$_maskbits" dev "$_iface"
 922                 return 1
 923                 ;;
 924         esac
 925     fi
 926 }
 927
 928 delete_ip_from_iface()
 929 {
 930     _iface=$1
 931     _ip=$2
 932     _maskbits=$3
 933
 934     # This could be set globally for all interfaces but it is probably
 935     # better to avoid surprises, so limit it the interfaces where CTDB
 936     # has public IP addresses.  There isn't anywhere else convenient
 937     # to do this so just set it each time.  This is much cheaper than
 938     # remembering and re-adding secondaries.
 939     set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
 940
 941     ip addr del "$_ip/$_maskbits" dev "$_iface" || {
 942         echo "Failed to del $_ip on dev $_iface"
 943         return 1
 944     }
 945 }
 946
 947 # If the given IP is hosted then print 2 items: maskbits and iface
 948 ip_maskbits_iface ()
 949 {
 950     _addr="$1"
 951
 952     case "$_addr" in
 953         *:*) _family="inet6" ; _bits=128 ;;
 954         *)   _family="inet"  ; _bits=32  ;;
 955     esac
 956
 957     ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
 958         awk -v family="${_family}" \
 959             'NR == 1 { iface = $2; sub(":$", "", iface) ; \
 960                        sub("@.*", "", iface) } \
 961              $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
 962                            print mask, iface, family }'
 963 }
 964
 965 drop_ip ()
 966 {
 967     _addr="${1%/*}"  # Remove optional maskbits
 968
 969     set -- $(ip_maskbits_iface $_addr)
 970     if [ -n "$1" ] ; then
 971         _maskbits="$1"
 972         _iface="$2"
 973         echo "Removing public address $_addr/$_maskbits from device $_iface"
 974         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
 975     fi
 976 }
 977
 978 drop_all_public_ips ()
 979 {
 980     while read _ip _x ; do
 981         drop_ip "$_ip"
 982     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
 983 }
 984
 985 flush_route_cache ()
 986 {
 987     set_proc_maybe sys/net/ipv4/route/flush 1
 988     set_proc_maybe sys/net/ipv6/route/flush 1
 989 }
 990
 991 ########################################################
 992 # Simple counters
 993 _ctdb_counter_common () {
 994     _service_name="${1:-${service_name:-${script_name}}}"
 995     _counter_file="$ctdb_fail_dir/$_service_name"
 996     mkdir -p "${_counter_file%/*}" # dirname
 997 }
 998 ctdb_counter_init () {
 999     _ctdb_counter_common "$1"
1000
1001     >"$_counter_file"
1002 }
1003 ctdb_counter_incr () {
1004     _ctdb_counter_common "$1"
1005
1006     # unary counting!
1007     echo -n 1 >> "$_counter_file"
1008 }
1009 ctdb_check_counter () {
1010     _msg="${1:-error}"  # "error"  - anything else is silent on fail
1011     _op="${2:--ge}"  # an integer operator supported by test
1012     _limit="${3:-${service_fail_limit}}"
1013     shift 3
1014     _ctdb_counter_common "$1"
1015
1016     # unary counting!
1017     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
1018     _hit=false
1019     if [ "$_op" != "%" ] ; then
1020         if [ $_size $_op $_limit ] ; then
1021             _hit=true
1022         fi
1023     else
1024         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
1025             _hit=true
1026         fi
1027     fi
1028     if $_hit ; then
1029         if [ "$_msg" = "error" ] ; then
1030             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1031             exit 1
1032         else
1033             return 1
1034         fi
1035     fi
1036 }
1037
1038 ########################################################
1039
1040 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1041 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1042
1043 ctdb_setup_service_state_dir ()
1044 {
1045     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1046     mkdir -p "$service_state_dir" || {
1047         echo "Error creating state dir \"$service_state_dir\""
1048         exit 1
1049     }
1050 }
1051
1052 ########################################################
1053 # Managed status history, for auto-start/stop
1054
1055 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1056
1057 _ctdb_managed_common ()
1058 {
1059     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1060 }
1061
1062 ctdb_service_managed ()
1063 {
1064     _ctdb_managed_common
1065     mkdir -p "$ctdb_managed_dir"
1066     touch "$_ctdb_managed_file"
1067 }
1068
1069 ctdb_service_unmanaged ()
1070 {
1071     _ctdb_managed_common
1072     rm -f "$_ctdb_managed_file"
1073 }
1074
1075 is_ctdb_previously_managed_service ()
1076 {
1077     _ctdb_managed_common
1078     [ -f "$_ctdb_managed_file" ]
1079 }
1080
1081 ########################################################
1082 # Check and set status
1083
1084 log_status_cat ()
1085 {
1086     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1087 }
1088
1089 ctdb_checkstatus ()
1090 {
1091     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1092         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1093         return 1
1094     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1095         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1096         return 2
1097     else
1098         return 0
1099     fi
1100 }
1101
1102 ctdb_setstatus ()
1103 {
1104     d="$ctdb_status_dir/$script_name"
1105     case "$1" in
1106         unhealthy|banned)
1107             mkdir -p "$d"
1108             cat "$2" >"$d/$1"
1109             ;;
1110         *)
1111             for i in "banned" "unhealthy" ; do
1112                 rm -f "$d/$i"
1113             done
1114             ;;
1115     esac
1116 }
1117
1118 ##################################################################
1119 # Reconfigure a service on demand
1120
1121 _ctdb_service_reconfigure_common ()
1122 {
1123     _d="$ctdb_status_dir/${service_name}"
1124     mkdir -p "$_d"
1125     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1126 }
1127
1128 ctdb_service_needs_reconfigure ()
1129 {
1130     _ctdb_service_reconfigure_common
1131     [ -e "$_ctdb_service_reconfigure_flag" ]
1132 }
1133
1134 ctdb_service_set_reconfigure ()
1135 {
1136     _ctdb_service_reconfigure_common
1137     >"$_ctdb_service_reconfigure_flag"
1138 }
1139
1140 ctdb_service_unset_reconfigure ()
1141 {
1142     _ctdb_service_reconfigure_common
1143     rm -f "$_ctdb_service_reconfigure_flag"
1144 }
1145
1146 ctdb_service_reconfigure ()
1147 {
1148     echo "Reconfiguring service \"${service_name}\"..."
1149     ctdb_service_unset_reconfigure
1150     service_reconfigure || return $?
1151     ctdb_counter_init
1152 }
1153
1154 # Default service_reconfigure() function does nothing.
1155 service_reconfigure ()
1156 {
1157     :
1158 }
1159
1160 ctdb_reconfigure_take_lock ()
1161 {
1162     _ctdb_service_reconfigure_common
1163     _lock="${_d}/reconfigure_lock"
1164     mkdir -p "${_lock%/*}" # dirname
1165     touch "$_lock"
1166
1167     (
1168         flock 0
1169         # This is overkill but will work if we need to extend this to
1170         # allow certain events to run multiple times in parallel
1171         # (e.g. takeip) and write multiple PIDs to the file.
1172         read _locker_event
1173         if [ -n "$_locker_event" ] ; then
1174             while read _pid ; do
1175                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1176                     kill -0 "$_pid" 2>/dev/null ; then
1177                     exit 1
1178                 fi
1179             done
1180         fi
1181
1182         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1183         exit 0
1184     ) <"$_lock"
1185 }
1186
1187 ctdb_reconfigure_release_lock ()
1188 {
1189     _ctdb_service_reconfigure_common
1190     _lock="${_d}/reconfigure_lock"
1191
1192     rm -f "$_lock"
1193 }
1194
1195 ctdb_replay_monitor_status ()
1196 {
1197     echo "Replaying previous status for this script due to reconfigure..."
1198     # Leading separator ('|') is missing in some versions...
1199     _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1200     # Output looks like this:
1201     # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1202     # This is the cheapest way of getting fields in the middle.
1203     set -- $(IFS="|" ; echo $_out)
1204     _code="$3"
1205     _status="$4"
1206     # The error output field can include colons so we'll try to
1207     # preserve them.  The weak checking at the beginning tries to make
1208     # this work for both broken (no leading '|') and fixed output.
1209     _out="${_out%|}"
1210     _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1211     case "$_status" in
1212         OK) : ;;  # Do nothing special.
1213         TIMEDOUT)
1214             # Recast this as an error, since we can't exit with the
1215             # correct negative number.
1216             _code=1
1217             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1218             ;;
1219         DISABLED)
1220             # Recast this as an OK, since we can't exit with the
1221             # correct negative number.
1222             _code=0
1223             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1224             ;;
1225         *) : ;;  # Must be ERROR, do nothing special.
1226     esac
1227     if [ -n "$_err_out" ] ; then
1228         echo "$_err_out"
1229     fi
1230     exit $_code
1231 }
1232
1233 ctdb_service_check_reconfigure ()
1234 {
1235     assert_service_name
1236
1237     # We only care about some events in this function.  For others we
1238     # return now.
1239     case "$event_name" in
1240         monitor|ipreallocated|reconfigure) : ;;
1241         *) return 0 ;;
1242     esac
1243
1244     if ctdb_reconfigure_take_lock ; then
1245         # No events covered by this function are running, so proceed
1246         # with gay abandon.
1247         case "$event_name" in
1248             reconfigure)
1249                 (ctdb_service_reconfigure)
1250                 exit $?
1251                 ;;
1252             ipreallocated)
1253                 if ctdb_service_needs_reconfigure ; then
1254                     ctdb_service_reconfigure
1255                 fi
1256                 ;;
1257         esac
1258
1259         ctdb_reconfigure_release_lock
1260     else
1261         # Somebody else is running an event we don't want to collide
1262         # with.  We proceed with caution.
1263         case "$event_name" in
1264             reconfigure)
1265                 # Tell whoever called us to retry.
1266                 exit 2
1267                 ;;
1268             ipreallocated)
1269                 # Defer any scheduled reconfigure and just run the
1270                 # rest of the ipreallocated event, as per the
1271                 # eventscript.  There's an assumption here that the
1272                 # event doesn't depend on any scheduled reconfigure.
1273                 # This is true in the current code.
1274                 return 0
1275                 ;;
1276             monitor)
1277                 # There is most likely a reconfigure in progress so
1278                 # the service is possibly unstable.  As above, we
1279                 # defer any scheduled reconfigured.  We also replay
1280                 # the previous monitor status since that's the best
1281                 # information we have.
1282                 ctdb_replay_monitor_status
1283                 ;;
1284         esac
1285     fi
1286 }
1287
1288 ##################################################################
1289 # Does CTDB manage this service? - and associated auto-start/stop
1290
1291 ctdb_compat_managed_service ()
1292 {
1293     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1294         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1295     fi
1296 }
1297
1298 is_ctdb_managed_service ()
1299 {
1300     assert_service_name
1301
1302     # $t is used just for readability and to allow better accurate
1303     # matching via leading/trailing spaces
1304     t=" $CTDB_MANAGED_SERVICES "
1305
1306     # Return 0 if "<space>$service_name<space>" appears in $t
1307     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1308         return 0
1309     fi
1310
1311     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1312     # backward compatibility and try again.
1313     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1314     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1315     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1316     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1317     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1318     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1319     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1320     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1321     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1322
1323     t=" $CTDB_MANAGED_SERVICES "
1324
1325     # Return 0 if "<space>$service_name<space>" appears in $t
1326     [ "${t#* ${service_name} }" != "${t}" ]
1327 }
1328
1329 ctdb_start_stop_service ()
1330 {
1331     assert_service_name
1332
1333     # Allow service-start/service-stop pseudo-events to start/stop
1334     # services when we're not auto-starting/stopping and we're not
1335     # monitoring.
1336     case "$event_name" in
1337         service-start)
1338             if is_ctdb_managed_service ; then
1339                 die 'service-start event not permitted when service is managed'
1340             fi
1341             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1342                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1343             fi
1344             ctdb_service_start
1345             exit $?
1346             ;;
1347         service-stop)
1348             if is_ctdb_managed_service ; then
1349                 die 'service-stop event not permitted when service is managed'
1350             fi
1351             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1352                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1353             fi
1354             ctdb_service_stop
1355             exit $?
1356             ;;
1357     esac
1358
1359     # Do nothing unless configured to...
1360     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1361
1362     [ "$event_name" = "monitor" ] || return 0
1363
1364     if is_ctdb_managed_service ; then
1365         if ! is_ctdb_previously_managed_service ; then
1366             echo "Starting service \"$service_name\" - now managed"
1367             background_with_logging ctdb_service_start
1368             exit $?
1369         fi
1370     else
1371         if is_ctdb_previously_managed_service ; then
1372             echo "Stopping service \"$service_name\" - no longer managed"
1373             background_with_logging ctdb_service_stop
1374             exit $?
1375         fi
1376     fi
1377 }
1378
1379 ctdb_service_start ()
1380 {
1381     # The service is marked managed if we've ever tried to start it.
1382     ctdb_service_managed
1383
1384     service_start || return $?
1385
1386     ctdb_counter_init
1387     ctdb_check_tcp_init
1388 }
1389
1390 ctdb_service_stop ()
1391 {
1392     ctdb_service_unmanaged
1393     service_stop
1394 }
1395
1396 # Default service_start() and service_stop() functions.
1397
1398 # These may be overridden in an eventscript.
1399 service_start ()
1400 {
1401     service "$service_name" start
1402 }
1403
1404 service_stop ()
1405 {
1406     service "$service_name" stop
1407 }
1408
1409 ##################################################################
1410
1411 ctdb_standard_event_handler ()
1412 {
1413     case "$1" in
1414         status)
1415             ctdb_checkstatus
1416             exit
1417             ;;
1418         setstatus)
1419             shift
1420             ctdb_setstatus "$@"
1421             exit
1422             ;;
1423     esac
1424 }
1425
1426 iptables_wrapper ()
1427 {
1428     _family="$1" ; shift
1429     if [ "$_family" = "inet6" ] ; then
1430         _iptables_cmd="ip6tables"
1431     else
1432         _iptables_cmd="iptables"
1433     fi
1434
1435     # iptables doesn't like being re-entered, so flock-wrap it.
1436     flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1437 }
1438
1439 # AIX (and perhaps others?) doesn't have mktemp
1440 if ! type mktemp >/dev/null 2>&1 ; then
1441     mktemp ()
1442     {
1443         _dir=false
1444         if [ "$1" = "-d" ] ; then
1445             _dir=true
1446             shift
1447         fi
1448         _d="${TMPDIR:-/tmp}"
1449         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1450             md5sum | \
1451             sed -e 's@\(..........\).*@\1@')
1452         _t="${_d}/tmp.${_hex10}"
1453         (
1454             umask 077
1455             if $_dir ; then
1456                 mkdir "$_t"
1457             else
1458                 >"$_t"
1459             fi
1460         )
1461         echo "$_t"
1462     }
1463 fi
1464
1465 ########################################################
1466 # tickle handling
1467 ########################################################
1468
1469 update_tickles ()
1470 {
1471         _port="$1"
1472
1473         tickledir="$CTDB_VARDIR/state/tickles"
1474         mkdir -p "$tickledir"
1475
1476         ctdb_get_pnn
1477
1478         # What public IPs do I hold?
1479         _ips=$(ctdb -X ip | awk -F'|' -v pnn=$pnn '$3 == pnn {print $2}')
1480
1481         # IPs as a regexp choice
1482         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1483
1484         # Record connections to our public IPs in a temporary file
1485         _my_connections="${tickledir}/${_port}.connections"
1486         rm -f "$_my_connections"
1487         netstat -tn |
1488         awk -v destpat="^${_ipschoice}:${_port}\$" \
1489           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1490         sort >"$_my_connections"
1491
1492         # Record our current tickles in a temporary file
1493         _my_tickles="${tickledir}/${_port}.tickles"
1494         rm -f "$_my_tickles"
1495         for _i in $_ips ; do
1496                 ctdb -X gettickles $_i $_port |
1497                 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1498         done |
1499         sort >"$_my_tickles"
1500
1501         # Add tickles for connections that we haven't already got tickles for
1502         comm -23 "$_my_connections" "$_my_tickles" |
1503         while read _src _dst ; do
1504                 ctdb addtickle $_src $_dst
1505         done
1506
1507         # Remove tickles for connections that are no longer there
1508         comm -13 "$_my_connections" "$_my_tickles" |
1509         while read _src _dst ; do
1510                 ctdb deltickle $_src $_dst
1511         done
1512
1513         rm -f "$_my_connections" "$_my_tickles"
1514 }
1515
1516 ########################################################
1517 # load a site local config file
1518 ########################################################
1519
1520 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1521         . "$CTDB_RC_LOCAL"
1522 }
1523
1524 [ -x $CTDB_BASE/rc.local ] && {
1525         . $CTDB_BASE/rc.local
1526 }
1527
1528 [ -d $CTDB_BASE/rc.local.d ] && {
1529         for i in $CTDB_BASE/rc.local.d/* ; do
1530                 [ -x "$i" ] && . "$i"
1531         done
1532 }
1533
1534 script_name="${0##*/}"       # basename
1535 service_fail_limit=1
1536 event_name="$1"