docs: insert meta data into kernel change notify parameter
[Samba.git] / ctdb / config / functions
blob1aad3ae79b32f08320214b6fe8dc2ea032db6685
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6     if [ -d "/var/lib/ctdb" ] ; then
7         export CTDB_VARDIR="/var/lib/ctdb"
8     else
9         export CTDB_VARDIR="/var/ctdb"
10     fi
12 [ -z "$CTDB_ETCDIR" ] && {
13     export CTDB_ETCDIR="/etc"
16 #######################################
17 # pull in a system config file, if any
18 _loadconfig() {
20     if [ -z "$1" ] ; then
21         foo="${service_config:-${service_name}}"
22         if [ -n "$foo" ] ; then
23             loadconfig "$foo"
24             return
25         fi
26     fi
28     if [ "$1" != "ctdb" ] ; then
29         loadconfig "ctdb"
30     fi
32     if [ -z "$1" ] ; then
33         return
34     fi
36     if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37         . $CTDB_ETCDIR/sysconfig/$1
38     elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39         . $CTDB_ETCDIR/default/$1
40     elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41         . $CTDB_BASE/sysconfig/$1
42     fi
44     if [ "$1" = "ctdb" ] ; then
45         _config="${CTDB_BASE}/ctdbd.conf"
46         if [ -r "$_config" ] ; then
47             . "$_config"
48         fi
49     fi
52 loadconfig () {
53     _loadconfig "$@"
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
59 # configuration file.
60 debug ()
62     if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63         # If there are arguments then echo them.  Otherwise expect to
64         # use stdin, which allows us to pass lots of debug using a
65         # here document.
66         if [ -n "$1" ] ; then
67             echo "DEBUG: $*"
68         elif ! tty -s ; then
69             sed -e 's@^@DEBUG: @'
70         fi
71     fi
74 die ()
76     _msg="$1"
77     _rc="${2:-1}"
79     echo "$_msg"
80     exit $_rc
83 # Log given message or stdin to either syslog or a CTDB log file
84 # $1 is the tag passed to logger if syslog is in use.
85 script_log ()
87     _tag="$1" ; shift
89     if [ "$CTDB_SYSLOG" = "yes" ] ; then
90         logger -t "ctdbd: ${_tag}" $*
91     else
92         {
93             if [ -n "$*" ] ; then
94                 echo "$*"
95             else
96                 cat
97             fi
98         } >>"${CTDB_LOGFILE:-/var/log/log.ctdb}"
99     fi
102 # When things are run in the background in an eventscript then logging
103 # output might get lost.  This is the "solution".  :-)
104 background_with_logging ()
106     (
107         "$@" 2>&1 </dev/null |
108         script_log "${script_name}&"
109     )&
111     return 0
114 ##############################################################
115 # check number of args for different events
116 ctdb_check_args ()
118     case "$1" in
119         takeip|releaseip)
120             if [ $# != 4 ]; then
121                 echo "ERROR: must supply interface, IP and maskbits"
122                 exit 1
123             fi
124             ;;
125         updateip)
126             if [ $# != 5 ]; then
127                 echo "ERROR: must supply old interface, new interface, IP and maskbits"
128                 exit 1
129             fi
130             ;;
131     esac
134 ##############################################################
135 # determine on what type of system (init style) we are running
136 detect_init_style()
138     # only do detection if not already set:
139     [ -z "$CTDB_INIT_STYLE" ] || return
141     if [ -x /sbin/startproc ]; then
142         CTDB_INIT_STYLE="suse"
143     elif [ -x /sbin/start-stop-daemon ]; then
144         CTDB_INIT_STYLE="debian"
145     else
146         CTDB_INIT_STYLE="redhat"
147     fi
150 ######################################################
151 # simulate /sbin/service on platforms that don't have it
152 # _service() makes it easier to hook the service() function for
153 # testing.
154 _service ()
156   _service_name="$1"
157   _op="$2"
159   # do nothing, when no service was specified
160   [ -z "$_service_name" ] && return
162   if [ -x /sbin/service ]; then
163       $_nice /sbin/service "$_service_name" "$_op"
164   elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
165       $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
166   elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
167       $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
168   fi
171 service()
173     _nice=""
174     _service "$@"
177 ######################################################
178 # simulate /sbin/service (niced) on platforms that don't have it
179 nice_service()
181     _nice="nice"
182     _service "$@"
185 ######################################################
186 # wrapper around /proc/ settings to allow them to be hooked
187 # for testing
188 # 1st arg is relative path under /proc/, 2nd arg is value to set
189 set_proc ()
191     echo "$2" >"/proc/$1"
194 ######################################################
195 # wrapper around getting file contents from /proc/ to allow
196 # this to be hooked for testing
197 # 1st arg is relative path under /proc/
198 get_proc ()
200     cat "/proc/$1"
203 ######################################################
204 # Check that an RPC service is healthy -
205 # this includes allowing a certain number of failures
206 # before marking the NFS service unhealthy.
208 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
210 # each triple is a set of 3 arguments: an operator, a 
211 # fail count limit and an action string.
213 # For example:
215 #       nfs_check_rpc_service "lockd" \
216 #           -ge 15 "verbose restart unhealthy" \
217 #           -eq 10 "restart:bs"
219 # says that if lockd is down for 15 iterations then do
220 # a verbose restart of lockd and mark the node unhealthy.
221 # Before this, after 10 iterations of failure, the
222 # service is restarted silently in the background.
223 # Order is important: the number of failures need to be
224 # specified in reverse order because processing stops
225 # after the first condition that is true.
226 ######################################################
227 nfs_check_rpc_service ()
229     _prog_name="$1" ; shift
231     if _nfs_check_rpc_common "$_prog_name" ; then
232         return
233     fi
235     while [ -n "$3" ] ; do
236         if _nfs_check_rpc_action "$1" "$2" "$3" ; then
237             break
238         fi
239         shift 3
240     done
243 # The new way of doing things...
244 nfs_check_rpc_services ()
246     # Files must end with .check - avoids editor backups, RPM fu, ...
247     for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
248         _t="${_f%.check}"
249         _prog_name="${_t##*/[0-9][0-9].}"
251         if _nfs_check_rpc_common "$_prog_name" ; then
252             # This RPC service is up, check next service...
253             continue
254         fi
256         # Check each line in the file in turn until one of the limit
257         # checks is hit...
258         while read _cmp _lim _rest ; do
259             # Skip comments
260             case "$_cmp" in
261                 \#*) continue ;;
262             esac
264             if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
265                 # Limit was hit on this line, no further checking...
266                 break
267             fi
268         done <"$_f"
269     done
272 _nfs_check_rpc_common ()
274     _prog_name="$1"
276     # Some platforms don't have separate programs for all services.
277     case "$_prog_name" in
278         statd)
279             which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
280     esac
282     case "$_prog_name" in
283         nfsd)
284             _rpc_prog=nfs
285             _version=3
286             ;;
287         mountd)
288             _rpc_prog=mountd
289             _version=1
290             ;;
291         rquotad)
292             _rpc_prog=rquotad
293             _version=1
294             ;;
295         lockd)
296             _rpc_prog=nlockmgr
297             _version=4
298             ;;
299         statd)
300             _rpc_prog=status
301             _version=1
302             ;;
303         *)
304             echo "Internal error: unknown RPC program \"$_prog_name\"."
305             exit 1
306     esac
308     _service_name="nfs_${_prog_name}"
310     if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
311         ctdb_counter_init "$_service_name"
312         return 0
313     fi
315     ctdb_counter_incr "$_service_name"
317     return 1
320 _nfs_check_rpc_action ()
322     _cmp="$1"
323     _limit="$2"
324     _actions="$3"
326     if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
327         return 1
328     fi
330     for _action in $_actions ; do
331         case "$_action" in
332             verbose)
333                 echo "$ctdb_check_rpc_out"
334                 ;;
335             restart)
336                 _nfs_restart_rpc_service "$_prog_name"
337                 ;;
338             restart:b)
339                 _nfs_restart_rpc_service "$_prog_name" true
340                 ;;
341             unhealthy)
342                 exit 1
343                 ;;
344             *)
345                 echo "Internal error: unknown action \"$_action\"."
346                 exit 1
347         esac
348     done
350     return 0
353 _nfs_restart_rpc_service ()
355     _prog_name="$1"
356     _background="${2:-false}"
358     if $_background ; then
359         _maybe_background="background_with_logging"
360     else
361         _maybe_background=""
362     fi
364     _p="rpc.${_prog_name}"
366     case "$_prog_name" in
367         nfsd)
368             echo "Trying to restart NFS service"
369             $_maybe_background startstop_nfs restart
370             ;;
371         mountd)
372             echo "Trying to restart $_prog_name [${_p}]"
373             killall -q -9 "$_p"
374             $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
375             ;;
376         rquotad)
377             echo "Trying to restart $_prog_name [${_p}]"
378             killall -q -9 "$_p"
379             $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
380             ;;
381         lockd)
382             echo "Trying to restart lock manager service"
383             $_maybe_background startstop_nfslock restart
384             ;;
385         statd)
386             echo "Trying to restart $_prog_name [${_p}]"
387             killall -q -9 "$_p"
388             $_maybe_background $_p \
389                 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
390                 ${STATD_PORT:+-p} $STATD_PORT \
391                 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
392             ;;
393         *)
394             echo "Internal error: unknown RPC program \"$_prog_name\"."
395             exit 1
396     esac
399 ######################################################
400 # check that a rpc server is registered with portmap
401 # and responding to requests
402 # usage: ctdb_check_rpc SERVICE_NAME VERSION
403 ######################################################
404 ctdb_check_rpc ()
406     progname="$1"
407     version="$2"
409     _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
411     if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
412         ctdb_check_rpc_out="ERROR: $progname failed RPC check:
413 $ctdb_check_rpc_out"
414         echo "$ctdb_check_rpc_out"
415         return 1
416     fi
419 ######################################################
420 # Ensure $service_name is set
421 assert_service_name ()
423     [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
426 ######################################################
427 # check a set of directories is available
428 # return 1 on a missing directory
429 # directories are read from stdin
430 ######################################################
431 ctdb_check_directories_probe()
433     while IFS="" read d ; do
434         case "$d" in
435             *%*)
436                 continue
437                 ;;
438             *)
439                 [ -d "${d}/." ] || return 1
440         esac
441     done
444 ######################################################
445 # check a set of directories is available
446 # directories are read from stdin
447 ######################################################
448 ctdb_check_directories()
450     ctdb_check_directories_probe || {
451         echo "ERROR: $service_name directory \"$d\" not available"
452         exit 1
453     }
456 ######################################################
457 # check a set of tcp ports
458 # usage: ctdb_check_tcp_ports <ports...>
459 ######################################################
461 # This flag file is created when a service is initially started.  It
462 # is deleted the first time TCP port checks for that service succeed.
463 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
464 # message if a port check fails.
465 _ctdb_check_tcp_common ()
467     assert_service_name
468     _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
471 ctdb_check_tcp_init ()
473     _ctdb_check_tcp_common
474     mkdir -p "${_ctdb_service_started_file%/*}" # dirname
475     touch "$_ctdb_service_started_file"
478 # Check whether something is listening on all of the given TCP ports
479 # using the "ctdb checktcpport" command.
480 ctdb_check_tcp_ports()
482     if [ -z "$1" ] ; then
483         echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
484         exit 1
485     fi
487     for _p ; do  # process each function argument (port)
488         _cmd="ctdb checktcpport $_p"
489         _out=$($_cmd 2>&1)
490         _ret=$?
491         case "$_ret" in
492             0)
493                 _ctdb_check_tcp_common
494                 if [ ! -f "$_ctdb_service_started_file" ] ; then
495                     echo "ERROR: $service_name tcp port $_p is not responding"
496                     debug "\"ctdb checktcpport $_p\" was able to bind to port"
497                 else
498                     echo "INFO: $service_name tcp port $_p is not responding"
499                 fi
501                 return 1
502                 ;;
503             98)
504                 # Couldn't bind, something already listening, next port...
505                 continue
506                 ;;
507             *)
508                 echo "ERROR: unexpected error running \"ctdb checktcpport\""
509                 debug <<EOF
510 ctdb checktcpport (exited with $_ret) with output:
511 $_out"
513                 return $_ret
514         esac
515     done
517     # All ports listening
518     _ctdb_check_tcp_common
519     rm -f "$_ctdb_service_started_file"
520     return 0
523 ######################################################
524 # check a unix socket
525 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
526 ######################################################
527 ctdb_check_unix_socket() {
528     socket_path="$1"
529     [ -z "$socket_path" ] && return
531     if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
532         echo "ERROR: $service_name socket $socket_path not found"
533         return 1
534     fi
537 ######################################################
538 # check a command returns zero status
539 # usage: ctdb_check_command <command>
540 ######################################################
541 ctdb_check_command ()
543     _out=$("$@" 2>&1) || {
544         echo "ERROR: $* returned error"
545         echo "$_out" | debug
546         exit 1
547     }
550 ################################################
551 # kill off any TCP connections with the given IP
552 ################################################
553 kill_tcp_connections ()
555     _ip="$1"
557     _oneway=false
558     if [ "$2" = "oneway" ] ; then
559         _oneway=true
560     fi
562     get_tcp_connections_for_ip "$_ip" | {
563         _killcount=0
564         _connections=""
565         _nl="
567         while read _dst _src; do
568             _destport="${_dst##*:}"
569             __oneway=$_oneway
570             case $_destport in
571                 # we only do one-way killtcp for CIFS
572                 139|445) __oneway=true ;;
573             esac
575             echo "Killing TCP connection $_src $_dst"
576             _connections="${_connections}${_nl}${_src} ${_dst}"
577             if ! $__oneway ; then
578                 _connections="${_connections}${_nl}${_dst} ${_src}"
579             fi
581             _killcount=$(($_killcount + 1))
582         done
584         if [ $_killcount -eq 0 ] ; then
585             return
586         fi
588         echo "$_connections" | ctdb killtcp || {
589             echo "Failed to send killtcp control"
590             return
591         }
593         _count=0
594         while : ; do
595             _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
597             if [ $_remaining -eq 0 ] ; then
598                 echo "Killed $_killcount TCP connections to released IP $_ip"
599                 return
600             fi
602             _count=$(($_count + 1))
603             if [ $_count -gt 3 ] ; then
604                 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
605                 return
606             fi
608             echo "Waiting for $_remaining connections to be killed for IP $_ip"
609             sleep 1
610         done
611     }
614 ##################################################################
615 # kill off the local end for any TCP connections with the given IP
616 ##################################################################
617 kill_tcp_connections_local_only ()
619     kill_tcp_connections "$1" "oneway"
622 ##################################################################
623 # tickle any TCP connections with the given IP
624 ##################################################################
625 tickle_tcp_connections ()
627     _ip="$1"
629     get_tcp_connections_for_ip "$_ip" |
630     {
631         _failed=false
633         while read dest src; do
634             echo "Tickle TCP connection $src $dest"
635             ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
636             echo "Tickle TCP connection $dest $src"
637             ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
638         done
640         if $_failed ; then
641             echo "Failed to send tickle control"
642         fi
643     }
646 get_tcp_connections_for_ip ()
648     _ip="$1"
650     netstat -tn | awk -v ip=$_ip \
651         'index($1, "tcp") == 1 && \
652          (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
653          && $6 == "ESTABLISHED" \
654          {print $4" "$5}'
657 ########################################################
658 # start/stop the Ganesha nfs service
659 ########################################################
660 startstop_ganesha()
662     _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
663     case "$1" in
664         start)
665             service "$_service_name" start
666             ;;
667         stop)
668             service "$_service_name" stop
669             ;;
670         restart)
671             service "$_service_name" restart
672             ;;
673     esac
676 ########################################################
677 # start/stop the nfs service on different platforms
678 ########################################################
679 startstop_nfs() {
680         PLATFORM="unknown"
681         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
682                 PLATFORM="sles"
683         }
684         [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
685                 PLATFORM="rhel"
686         }
688         case $PLATFORM in
689         sles)
690                 case $1 in
691                 start)
692                         service nfsserver start
693                         ;;
694                 stop)
695                         service nfsserver stop > /dev/null 2>&1
696                         ;;
697                 restart)
698                         set_proc "fs/nfsd/threads" 0
699                         service nfsserver stop > /dev/null 2>&1
700                         pkill -9 nfsd
701                         nfs_dump_some_threads
702                         service nfsserver start
703                         ;;
704                 esac
705                 ;;
706         rhel)
707                 case $1 in
708                 start)
709                         service nfslock start
710                         service nfs start
711                         ;;
712                 stop)
713                         service nfs stop
714                         service nfslock stop
715                         ;;
716                 restart)
717                         set_proc "fs/nfsd/threads" 0
718                         service nfs stop > /dev/null 2>&1
719                         service nfslock stop > /dev/null 2>&1
720                         pkill -9 nfsd
721                         nfs_dump_some_threads
722                         service nfslock start
723                         service nfs start
724                         ;;
725                 esac
726                 ;;
727         *)
728                 echo "Unknown platform. NFS is not supported with ctdb"
729                 exit 1
730                 ;;
731         esac
734 # Dump up to the configured number of nfsd thread backtraces.
735 nfs_dump_some_threads ()
737     [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] || return 0
739     # Optimisation to avoid running an unnecessary pidof
740     [ $CTDB_NFS_DUMP_STUCK_THREADS -gt 0 ] || return 0
742     _count=0
743     for _pid in $(pidof nfsd) ; do
744         [ $_count -le $CTDB_NFS_DUMP_STUCK_THREADS ] || break
746         # Do this first to avoid racing with thread exit
747         _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
748         if [ -n "$_stack" ] ; then
749             echo "Stack trace for stuck nfsd thread [${_pid}]:"
750             echo "$_stack"
751             _count=$(($_count + 1))
752         fi
753     done
756 ########################################################
757 # start/stop the nfs lockmanager service on different platforms
758 ########################################################
759 startstop_nfslock() {
760         PLATFORM="unknown"
761         [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
762                 PLATFORM="sles"
763         }
764         [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
765                 PLATFORM="rhel"
766         }
768         case $PLATFORM in
769         sles)
770                 # for sles there is no service for lockmanager
771                 # so we instead just shutdown/restart nfs
772                 case $1 in
773                 start)
774                         service nfsserver start
775                         ;;
776                 stop)
777                         service nfsserver stop > /dev/null 2>&1
778                         ;;
779                 restart)
780                         service nfsserver stop > /dev/null 2>&1
781                         service nfsserver start
782                         ;;
783                 esac
784                 ;;
785         rhel)
786                 case $1 in
787                 start)
788                         service nfslock start
789                         ;;
790                 stop)
791                         service nfslock stop > /dev/null 2>&1
792                         ;;
793                 restart)
794                         service nfslock stop > /dev/null 2>&1
795                         service nfslock start
796                         ;;
797                 esac
798                 ;;
799         *)
800                 echo "Unknown platform. NFS locking is not supported with ctdb"
801                 exit 1
802                 ;;
803         esac
806 # Periodically update the statd database
807 nfs_statd_update ()
809     _update_period="$1"
811     _statd_update_trigger="$service_state_dir/update-trigger"
812     [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
814     _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
815     _current_time=$(date +"%s")
816     if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
817         touch "$_statd_update_trigger"
818         $CTDB_BASE/statd-callout updatelocal &
819         $CTDB_BASE/statd-callout updateremote &
820     fi
823 add_ip_to_iface()
825     _iface=$1
826     _ip=$2
827     _maskbits=$3
829     _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
830     mkdir -p "${_lockfile%/*}" # dirname
831     [ -f "$_lockfile" ] || touch "$_lockfile"
833     (
834         # Note: use of return/exit/die() below only gets us out of the
835         # sub-shell, which is actually what we want.  That is, the
836         # function should just return non-zero.
838         flock --timeout 30 0 || \
839             die "add_ip_to_iface: unable to get lock for ${_iface}"
841         # Ensure interface is up
842         ip link set "$_iface" up || \
843             die "Failed to bringup interface $_iface"
845         ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || \
846             die "Failed to add $_ip/$_maskbits on dev $_iface"
847     ) <"$_lockfile"
849     # Do nothing here - return above only gets us out of the subshell
850     # and doing anything here will affect the return code.
853 delete_ip_from_iface()
855     _iface=$1
856     _ip=$2
857     _maskbits=$3
859     _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
860     mkdir -p "${_lockfile%/*}" # dirname
861     [ -f "$_lockfile" ] || touch "$_lockfile"
863     (
864         # Note: use of return/exit/die() below only gets us out of the
865         # sub-shell, which is actually what we want.  That is, the
866         # function should just return non-zero.
868         flock --timeout 30 0 || \
869             die "delete_ip_from_iface: unable to get lock for ${_iface}"
871         _im="$_ip/$_maskbits"  # shorthand for readability
873         # "ip addr del" will delete all secondary IPs if this is the
874         # primary.  To work around this _very_ annoying behaviour we
875         # have to keep a record of the secondaries and re-add them
876         # afterwards.  Yuck!
878         _secondaries=""
879         if ip addr list dev "$_iface" primary | grep -Fq "inet $_im " ; then
880             _secondaries=$(ip addr list dev "$_iface" secondary | \
881                 awk '$1 == "inet" { print $2 }')
882         fi
884         local _rc=0
885         ip addr del "$_im" dev "$_iface" || {
886             echo "Failed to del $_ip on dev $_iface"
887             _rc=1
888         }
890         if [ -n "$_secondaries" ] ; then
891             for _i in $_secondaries; do
892                 if ip addr list dev "$_iface" | grep -Fq "inet $_i" ; then
893                     echo "Kept secondary $_i on dev $_iface"
894                 else
895                     echo "Re-adding secondary address $_i to dev $_iface"
896                     ip addr add $_i brd + dev $_iface || {
897                         echo "Failed to re-add address $_i to dev $_iface"
898                         _rc=1
899                     }
900                 fi
901             done
902         fi
904         return $_rc
905     ) <"$_lockfile"
907     # Do nothing here - return above only gets us out of the subshell
908     # and doing anything here will affect the return code.
911 # If the given IP is hosted then print 2 items: maskbits and iface 
912 ip_maskbits_iface ()
914     _addr="$1"
916     ip addr show to "${_addr}/32" 2>/dev/null | \
917         awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }'
920 drop_ip ()
922     _addr="${1%/*}"  # Remove optional maskbits
924     set -- $(ip_maskbits_iface $_addr)
925     if [ -n "$1" ] ; then
926         _maskbits="$1"
927         _iface="$2"
928         echo "Removing public address $_addr/$_maskbits from device $_iface"
929         delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
930     fi
933 drop_all_public_ips ()
935     while read _ip _x ; do
936         drop_ip "$_ip"
937     done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
940 ########################################################
941 # Simple counters
942 _ctdb_counter_common () {
943     _service_name="${1:-${service_name:-${script_name}}}"
944     _counter_file="$ctdb_fail_dir/$_service_name"
945     mkdir -p "${_counter_file%/*}" # dirname
947 ctdb_counter_init () {
948     _ctdb_counter_common "$1"
950     >"$_counter_file"
952 ctdb_counter_incr () {
953     _ctdb_counter_common "$1"
955     # unary counting!
956     echo -n 1 >> "$_counter_file"
958 ctdb_check_counter () {
959     _msg="${1:-error}"  # "error"  - anything else is silent on fail
960     _op="${2:--ge}"  # an integer operator supported by test
961     _limit="${3:-${service_fail_limit}}"
962     shift 3
963     _ctdb_counter_common "$1"
965     # unary counting!
966     _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
967     _hit=false
968     if [ "$_op" != "%" ] ; then
969         if [ $_size $_op $_limit ] ; then
970             _hit=true
971         fi
972     else
973         if [ $(($_size $_op $_limit)) -eq 0 ] ; then
974             _hit=true
975         fi
976     fi
977     if $_hit ; then
978         if [ "$_msg" = "error" ] ; then
979             echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
980             exit 1              
981         else
982             return 1
983         fi
984     fi
987 ########################################################
989 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
990 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
992 ctdb_setup_service_state_dir ()
994     service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
995     mkdir -p "$service_state_dir" || {
996         echo "Error creating state dir \"$service_state_dir\""
997         exit 1
998     }
1001 ########################################################
1002 # Managed status history, for auto-start/stop
1004 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1006 _ctdb_managed_common ()
1008     _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1011 ctdb_service_managed ()
1013     _ctdb_managed_common
1014     mkdir -p "$ctdb_managed_dir"
1015     touch "$_ctdb_managed_file"
1018 ctdb_service_unmanaged ()
1020     _ctdb_managed_common
1021     rm -f "$_ctdb_managed_file"
1024 is_ctdb_previously_managed_service ()
1026     _ctdb_managed_common
1027     [ -f "$_ctdb_managed_file" ]
1030 ########################################################
1031 # Check and set status
1033 log_status_cat ()
1035     echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1038 ctdb_checkstatus ()
1040     if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1041         log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1042         return 1
1043     elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1044         log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1045         return 2
1046     else
1047         return 0
1048     fi
1051 ctdb_setstatus ()
1053     d="$ctdb_status_dir/$script_name"
1054     case "$1" in
1055         unhealthy|banned)
1056             mkdir -p "$d"
1057             cat "$2" >"$d/$1"
1058             ;;
1059         *)
1060             for i in "banned" "unhealthy" ; do
1061                 rm -f "$d/$i"
1062             done
1063             ;;
1064     esac
1067 ##################################################################
1068 # Reconfigure a service on demand
1070 _ctdb_service_reconfigure_common ()
1072     _d="$ctdb_status_dir/${service_name}"
1073     mkdir -p "$_d"
1074     _ctdb_service_reconfigure_flag="$_d/reconfigure"
1077 ctdb_service_needs_reconfigure ()
1079     _ctdb_service_reconfigure_common
1080     [ -e "$_ctdb_service_reconfigure_flag" ]
1083 ctdb_service_set_reconfigure ()
1085     _ctdb_service_reconfigure_common
1086     >"$_ctdb_service_reconfigure_flag"
1089 ctdb_service_unset_reconfigure ()
1091     _ctdb_service_reconfigure_common
1092     rm -f "$_ctdb_service_reconfigure_flag"
1095 ctdb_service_reconfigure ()
1097     echo "Reconfiguring service \"${service_name}\"..."
1098     ctdb_service_unset_reconfigure
1099     service_reconfigure || return $?
1100     ctdb_counter_init
1103 # Default service_reconfigure() function does nothing.
1104 service_reconfigure ()
1106     :
1109 ctdb_reconfigure_take_lock ()
1111     _ctdb_service_reconfigure_common
1112     _lock="${_d}/reconfigure_lock"
1113     mkdir -p "${_lock%/*}" # dirname
1114     touch "$_lock"
1116     (
1117         flock 0
1118         # This is overkill but will work if we need to extend this to
1119         # allow certain events to run multiple times in parallel
1120         # (e.g. takeip) and write multiple PIDs to the file.
1121         read _locker_event 
1122         if [ -n "$_locker_event" ] ; then
1123             while read _pid ; do
1124                 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1125                     kill -0 "$_pid" 2>/dev/null ; then
1126                     exit 1
1127                 fi
1128             done
1129         fi
1131         printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1132         exit 0
1133     ) <"$_lock"
1136 ctdb_reconfigure_release_lock ()
1138     _ctdb_service_reconfigure_common
1139     _lock="${_d}/reconfigure_lock"
1141     rm -f "$_lock"
1144 ctdb_replay_monitor_status ()
1146     echo "Replaying previous status for this script due to reconfigure..."
1147     # Leading colon (':') is missing in some versions...
1148     _out=$(ctdb scriptstatus -Y | grep -E "^:?monitor:${script_name}:")
1149     # Output looks like this:
1150     # :monitor:60.nfs:1:ERROR:1314764004.030861:1314764004.035514:foo bar:
1151     # This is the cheapest way of getting fields in the middle.
1152     set -- $(IFS=":" ; echo $_out)
1153     _code="$3"
1154     _status="$4"
1155     # The error output field can include colons so we'll try to
1156     # preserve them.  The weak checking at the beginning tries to make
1157     # this work for both broken (no leading ':') and fixed output.
1158     _out="${_out%:}"
1159     _err_out="${_out#*monitor:${script_name}:*:*:*:*:}"
1160     case "$_status" in
1161         OK) : ;;  # Do nothing special.
1162         TIMEDOUT)
1163             # Recast this as an error, since we can't exit with the
1164             # correct negative number.
1165             _code=1
1166             _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1167             ;;
1168         DISABLED)
1169             # Recast this as an OK, since we can't exit with the
1170             # correct negative number.
1171             _code=0
1172             _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1173             ;;
1174         *) : ;;  # Must be ERROR, do nothing special.
1175     esac
1176     if [ -n "$_err_out" ] ; then
1177         echo "$_err_out"
1178     fi
1179     exit $_code
1182 ctdb_service_check_reconfigure ()
1184     assert_service_name
1186     # We only care about some events in this function.  For others we
1187     # return now.
1188     case "$event_name" in
1189         monitor|ipreallocated|reconfigure) : ;;
1190         *) return 0 ;;
1191     esac
1193     if ctdb_reconfigure_take_lock ; then
1194         # No events covered by this function are running, so proceed
1195         # with gay abandon.
1196         case "$event_name" in
1197             reconfigure)
1198                 (ctdb_service_reconfigure)
1199                 exit $?
1200                 ;;
1201             ipreallocated)
1202                 if ctdb_service_needs_reconfigure ; then
1203                     ctdb_service_reconfigure
1204                 fi
1205                 ;;
1206         esac
1208         ctdb_reconfigure_release_lock
1209     else
1210         # Somebody else is running an event we don't want to collide
1211         # with.  We proceed with caution.
1212         case "$event_name" in
1213             reconfigure)
1214                 # Tell whoever called us to retry.
1215                 exit 2
1216                 ;;
1217             ipreallocated)
1218                 # Defer any scheduled reconfigure and just run the
1219                 # rest of the ipreallocated event, as per the
1220                 # eventscript.  There's an assumption here that the
1221                 # event doesn't depend on any scheduled reconfigure.
1222                 # This is true in the current code.
1223                 return 0
1224                 ;;
1225             monitor)
1226                 # There is most likely a reconfigure in progress so
1227                 # the service is possibly unstable.  As above, we
1228                 # defer any scheduled reconfigured.  We also replay
1229                 # the previous monitor status since that's the best
1230                 # information we have.
1231                 ctdb_replay_monitor_status
1232                 ;;
1233         esac
1234     fi
1237 ##################################################################
1238 # Does CTDB manage this service? - and associated auto-start/stop
1240 ctdb_compat_managed_service ()
1242     if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1243         CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1244     fi
1247 is_ctdb_managed_service ()
1249     assert_service_name
1251     # $t is used just for readability and to allow better accurate
1252     # matching via leading/trailing spaces
1253     t=" $CTDB_MANAGED_SERVICES "
1255     # Return 0 if "<space>$service_name<space>" appears in $t
1256     if [ "${t#* ${service_name} }" != "${t}" ] ; then
1257         return 0
1258     fi
1260     # If above didn't match then update $CTDB_MANAGED_SERVICES for
1261     # backward compatibility and try again.
1262     ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
1263     ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
1264     ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
1265     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
1266     ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
1267     ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
1268     ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
1269     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"
1270     ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs-ganesha-gpfs"
1272     t=" $CTDB_MANAGED_SERVICES "
1274     # Return 0 if "<space>$service_name<space>" appears in $t
1275     [ "${t#* ${service_name} }" != "${t}" ]
1278 ctdb_start_stop_service ()
1280     assert_service_name
1282     # Allow service-start/service-stop pseudo-events to start/stop
1283     # services when we're not auto-starting/stopping and we're not
1284     # monitoring.
1285     case "$event_name" in
1286         service-start)
1287             if is_ctdb_managed_service ; then
1288                 die 'service-start event not permitted when service is managed'
1289             fi
1290             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1291                 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1292             fi
1293             ctdb_service_start
1294             exit $?
1295             ;;
1296         service-stop)
1297             if is_ctdb_managed_service ; then
1298                 die 'service-stop event not permitted when service is managed'
1299             fi
1300             if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1301                 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1302             fi
1303             ctdb_service_stop
1304             exit $?
1305             ;;
1306     esac
1308     # Do nothing unless configured to...
1309     [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1311     [ "$event_name" = "monitor" ] || return 0
1313     if is_ctdb_managed_service ; then
1314         if ! is_ctdb_previously_managed_service ; then
1315             echo "Starting service \"$service_name\" - now managed"
1316             background_with_logging ctdb_service_start
1317             exit $?
1318         fi
1319     else
1320         if is_ctdb_previously_managed_service ; then
1321             echo "Stopping service \"$service_name\" - no longer managed"
1322             background_with_logging ctdb_service_stop
1323             exit $?
1324         fi
1325     fi
1328 ctdb_service_start ()
1330     # The service is marked managed if we've ever tried to start it.
1331     ctdb_service_managed
1333     service_start || return $?
1335     ctdb_counter_init
1336     ctdb_check_tcp_init
1339 ctdb_service_stop ()
1341     ctdb_service_unmanaged
1342     service_stop
1345 # Default service_start() and service_stop() functions.
1347 # These may be overridden in an eventscript.  When overriding, the
1348 # following convention must be followed.  If these functions are
1349 # called with no arguments then they may use internal logic to
1350 # determine whether the service is managed and, therefore, whether
1351 # they should take any action.  However, if the service name is
1352 # specified as an argument then an attempt must be made to start or
1353 # stop the service.  This is because the auto-start/stop code calls
1354 # them with the service name as an argument.
1355 service_start ()
1357     service "$service_name" start
1360 service_stop ()
1362     service "$service_name" stop
1365 ##################################################################
1367 ctdb_standard_event_handler ()
1369     case "$1" in
1370         status)
1371             ctdb_checkstatus
1372             exit
1373             ;;
1374         setstatus)
1375             shift
1376             ctdb_setstatus "$@"
1377             exit
1378             ;;
1379     esac
1382 # iptables doesn't like being re-entered, so flock-wrap it.
1383 iptables()
1385         flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1388 # AIX (and perhaps others?) doesn't have mktemp
1389 if ! which mktemp >/dev/null 2>&1 ; then
1390     mktemp ()
1391     {
1392         _dir=false
1393         if [ "$1" = "-d" ] ; then
1394             _dir=true
1395             shift
1396         fi
1397         _d="${TMPDIR:-/tmp}"
1398         _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1399             md5sum | \
1400             sed -e 's@\(..........\).*@\1@')
1401         _t="${_d}/tmp.${_hex10}"
1402         (
1403             umask 077
1404             if $_dir ; then
1405                 mkdir "$_t"
1406             else
1407                 >"$_t"
1408             fi
1409         )
1410         echo "$_t"
1411     }
1414 ########################################################
1415 # tickle handling
1416 ########################################################
1418 update_tickles ()
1420         _port="$1"
1422         tickledir="$CTDB_VARDIR/state/tickles"
1423         mkdir -p "$tickledir"
1425         # Who am I?
1426         _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1428         # What public IPs do I hold?
1429         _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
1431         # IPs as a regexp choice
1432         _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1434         # Record connections to our public IPs in a temporary file
1435         _my_connections="${tickledir}/${_port}.connections"
1436         rm -f "$_my_connections"
1437         netstat -tn |
1438         awk -v destpat="^${_ipschoice}:${_port}\$" \
1439           '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1440         sort >"$_my_connections"
1442         # Record our current tickles in a temporary file
1443         _my_tickles="${tickledir}/${_port}.tickles"
1444         rm -f "$_my_tickles"
1445         for _i in $_ips ; do
1446                 ctdb -Y gettickles $_i $_port | 
1447                 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1448         done |
1449         sort >"$_my_tickles"
1451         # Add tickles for connections that we haven't already got tickles for
1452         comm -23 "$_my_connections" "$_my_tickles" |
1453         while read _src _dst ; do
1454                 ctdb addtickle $_src $_dst
1455         done
1457         # Remove tickles for connections that are no longer there
1458         comm -13 "$_my_connections" "$_my_tickles" |
1459         while read _src _dst ; do
1460                 ctdb deltickle $_src $_dst
1461         done
1463         rm -f "$_my_connections" "$_my_tickles" 
1466 ########################################################
1467 # load a site local config file
1468 ########################################################
1470 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1471         . "$CTDB_RC_LOCAL"
1474 [ -x $CTDB_BASE/rc.local ] && {
1475         . $CTDB_BASE/rc.local
1478 [ -d $CTDB_BASE/rc.local.d ] && {
1479         for i in $CTDB_BASE/rc.local.d/* ; do
1480                 [ -x "$i" ] && . "$i"
1481         done
1484 script_name="${0##*/}"       # basename
1485 service_fail_limit=1
1486 event_name="$1"