1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6 if [ -d "/var/lib/ctdb" ] ; then
7 export CTDB_VARDIR="/var/lib/ctdb"
9 export CTDB_VARDIR="/var/ctdb"
12 [ -z "$CTDB_ETCDIR" ] && {
13 export CTDB_ETCDIR="/etc"
16 #######################################
17 # pull in a system config file, if any
21 foo="${service_config:-${service_name}}"
22 if [ -n "$foo" ] ; then
28 if [ "$1" != "ctdb" ] ; then
36 if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37 . $CTDB_ETCDIR/sysconfig/$1
38 elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39 . $CTDB_ETCDIR/default/$1
40 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41 . $CTDB_BASE/sysconfig/$1
44 if [ "$1" = "ctdb" ] ; then
45 _config="${CTDB_BASE}/ctdbd.conf"
46 if [ -r "$_config" ] ; then
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
62 if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63 # If there are arguments then echo them. Otherwise expect to
64 # use stdin, which allows us to pass lots of debug using a
83 # Log given message or stdin to either syslog or a CTDB log file
84 # $1 is the tag passed to logger if syslog is in use.
89 if [ "$CTDB_SYSLOG" = "yes" ] ; then
90 logger -t "ctdbd: ${_tag}" $*
98 } >>"${CTDB_LOGFILE:-/var/log/log.ctdb}"
102 # When things are run in the background in an eventscript then logging
103 # output might get lost. This is the "solution". :-)
104 background_with_logging ()
107 "$@" 2>&1 </dev/null |
108 script_log "${script_name}&"
114 ##############################################################
115 # check number of args for different events
121 echo "ERROR: must supply interface, IP and maskbits"
127 echo "ERROR: must supply old interface, new interface, IP and maskbits"
134 ##############################################################
135 # determine on what type of system (init style) we are running
138 # only do detection if not already set:
139 [ -z "$CTDB_INIT_STYLE" ] || return
141 if [ -x /sbin/startproc ]; then
142 CTDB_INIT_STYLE="suse"
143 elif [ -x /sbin/start-stop-daemon ]; then
144 CTDB_INIT_STYLE="debian"
146 CTDB_INIT_STYLE="redhat"
150 ######################################################
151 # simulate /sbin/service on platforms that don't have it
152 # _service() makes it easier to hook the service() function for
159 # do nothing, when no service was specified
160 [ -z "$_service_name" ] && return
162 if [ -x /sbin/service ]; then
163 $_nice /sbin/service "$_service_name" "$_op"
164 elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
165 $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
166 elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
167 $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
177 ######################################################
178 # simulate /sbin/service (niced) on platforms that don't have it
185 ######################################################
186 # wrapper around /proc/ settings to allow them to be hooked
188 # 1st arg is relative path under /proc/, 2nd arg is value to set
191 echo "$2" >"/proc/$1"
194 ######################################################
195 # wrapper around getting file contents from /proc/ to allow
196 # this to be hooked for testing
197 # 1st arg is relative path under /proc/
203 ######################################################
204 # Check that an RPC service is healthy -
205 # this includes allowing a certain number of failures
206 # before marking the NFS service unhealthy.
208 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
210 # each triple is a set of 3 arguments: an operator, a
211 # fail count limit and an action string.
215 # nfs_check_rpc_service "lockd" \
216 # -ge 15 "verbose restart unhealthy" \
217 # -eq 10 "restart:bs"
219 # says that if lockd is down for 15 iterations then do
220 # a verbose restart of lockd and mark the node unhealthy.
221 # Before this, after 10 iterations of failure, the
222 # service is restarted silently in the background.
223 # Order is important: the number of failures need to be
224 # specified in reverse order because processing stops
225 # after the first condition that is true.
226 ######################################################
227 nfs_check_rpc_service ()
229 _prog_name="$1" ; shift
231 if _nfs_check_rpc_common "$_prog_name" ; then
235 while [ -n "$3" ] ; do
236 if _nfs_check_rpc_action "$1" "$2" "$3" ; then
243 # The new way of doing things...
244 nfs_check_rpc_services ()
246 # Files must end with .check - avoids editor backups, RPM fu, ...
247 for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
249 _prog_name="${_t##*/[0-9][0-9].}"
251 if _nfs_check_rpc_common "$_prog_name" ; then
252 # This RPC service is up, check next service...
256 # Check each line in the file in turn until one of the limit
258 while read _cmp _lim _rest ; do
264 if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
265 # Limit was hit on this line, no further checking...
272 _nfs_check_rpc_common ()
276 # Some platforms don't have separate programs for all services.
277 case "$_prog_name" in
279 which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
282 case "$_prog_name" in
304 echo "Internal error: unknown RPC program \"$_prog_name\"."
308 _service_name="nfs_${_prog_name}"
310 if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
311 ctdb_counter_init "$_service_name"
315 ctdb_counter_incr "$_service_name"
320 _nfs_check_rpc_action ()
326 if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
330 for _action in $_actions ; do
333 echo "$ctdb_check_rpc_out"
336 _nfs_restart_rpc_service "$_prog_name"
339 _nfs_restart_rpc_service "$_prog_name" true
345 echo "Internal error: unknown action \"$_action\"."
353 _nfs_restart_rpc_service ()
356 _background="${2:-false}"
358 if $_background ; then
359 _maybe_background="background_with_logging"
364 _p="rpc.${_prog_name}"
366 case "$_prog_name" in
368 echo "Trying to restart NFS service"
369 $_maybe_background startstop_nfs restart
372 echo "Trying to restart $_prog_name [${_p}]"
374 $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
377 echo "Trying to restart $_prog_name [${_p}]"
379 $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
382 echo "Trying to restart lock manager service"
383 $_maybe_background startstop_nfslock restart
386 echo "Trying to restart $_prog_name [${_p}]"
388 $_maybe_background $_p \
389 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
390 ${STATD_PORT:+-p} $STATD_PORT \
391 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
394 echo "Internal error: unknown RPC program \"$_prog_name\"."
399 ######################################################
400 # check that a rpc server is registered with portmap
401 # and responding to requests
402 # usage: ctdb_check_rpc SERVICE_NAME VERSION
403 ######################################################
409 _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
411 if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
412 ctdb_check_rpc_out="ERROR: $progname failed RPC check:
414 echo "$ctdb_check_rpc_out"
419 ######################################################
420 # Ensure $service_name is set
421 assert_service_name ()
423 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
426 ######################################################
427 # check a set of directories is available
428 # return 1 on a missing directory
429 # directories are read from stdin
430 ######################################################
431 ctdb_check_directories_probe()
433 while IFS="" read d ; do
439 [ -d "${d}/." ] || return 1
444 ######################################################
445 # check a set of directories is available
446 # directories are read from stdin
447 ######################################################
448 ctdb_check_directories()
450 ctdb_check_directories_probe || {
451 echo "ERROR: $service_name directory \"$d\" not available"
456 ######################################################
457 # check a set of tcp ports
458 # usage: ctdb_check_tcp_ports <ports...>
459 ######################################################
461 # This flag file is created when a service is initially started. It
462 # is deleted the first time TCP port checks for that service succeed.
463 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
464 # message if a port check fails.
465 _ctdb_check_tcp_common ()
468 _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
471 ctdb_check_tcp_init ()
473 _ctdb_check_tcp_common
474 mkdir -p "${_ctdb_service_started_file%/*}" # dirname
475 touch "$_ctdb_service_started_file"
478 # Check whether something is listening on all of the given TCP ports
479 # using the "ctdb checktcpport" command.
480 ctdb_check_tcp_ports()
482 if [ -z "$1" ] ; then
483 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
487 for _p ; do # process each function argument (port)
488 _cmd="ctdb checktcpport $_p"
493 _ctdb_check_tcp_common
494 if [ ! -f "$_ctdb_service_started_file" ] ; then
495 echo "ERROR: $service_name tcp port $_p is not responding"
496 debug "\"ctdb checktcpport $_p\" was able to bind to port"
498 echo "INFO: $service_name tcp port $_p is not responding"
504 # Couldn't bind, something already listening, next port...
508 echo "ERROR: unexpected error running \"ctdb checktcpport\""
510 ctdb checktcpport (exited with $_ret) with output:
517 # All ports listening
518 _ctdb_check_tcp_common
519 rm -f "$_ctdb_service_started_file"
523 ######################################################
524 # check a unix socket
525 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
526 ######################################################
527 ctdb_check_unix_socket() {
529 [ -z "$socket_path" ] && return
531 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
532 echo "ERROR: $service_name socket $socket_path not found"
537 ######################################################
538 # check a command returns zero status
539 # usage: ctdb_check_command <command>
540 ######################################################
541 ctdb_check_command ()
543 _out=$("$@" 2>&1) || {
544 echo "ERROR: $* returned error"
550 ################################################
551 # kill off any TCP connections with the given IP
552 ################################################
553 kill_tcp_connections ()
558 if [ "$2" = "oneway" ] ; then
562 get_tcp_connections_for_ip "$_ip" | {
567 while read _dst _src; do
568 _destport="${_dst##*:}"
571 # we only do one-way killtcp for CIFS
572 139|445) __oneway=true ;;
575 echo "Killing TCP connection $_src $_dst"
576 _connections="${_connections}${_nl}${_src} ${_dst}"
577 if ! $__oneway ; then
578 _connections="${_connections}${_nl}${_dst} ${_src}"
581 _killcount=$(($_killcount + 1))
584 if [ $_killcount -eq 0 ] ; then
588 echo "$_connections" | ctdb killtcp || {
589 echo "Failed to send killtcp control"
595 _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
597 if [ $_remaining -eq 0 ] ; then
598 echo "Killed $_killcount TCP connections to released IP $_ip"
602 _count=$(($_count + 1))
603 if [ $_count -gt 3 ] ; then
604 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
608 echo "Waiting for $_remaining connections to be killed for IP $_ip"
614 ##################################################################
615 # kill off the local end for any TCP connections with the given IP
616 ##################################################################
617 kill_tcp_connections_local_only ()
619 kill_tcp_connections "$1" "oneway"
622 ##################################################################
623 # tickle any TCP connections with the given IP
624 ##################################################################
625 tickle_tcp_connections ()
629 get_tcp_connections_for_ip "$_ip" |
633 while read dest src; do
634 echo "Tickle TCP connection $src $dest"
635 ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
636 echo "Tickle TCP connection $dest $src"
637 ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
641 echo "Failed to send tickle control"
646 get_tcp_connections_for_ip ()
650 netstat -tn | awk -v ip=$_ip \
651 'index($1, "tcp") == 1 && \
652 (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
653 && $6 == "ESTABLISHED" \
657 ########################################################
658 # start/stop the Ganesha nfs service
659 ########################################################
662 _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
665 service "$_service_name" start
668 service "$_service_name" stop
671 service "$_service_name" restart
676 ########################################################
677 # start/stop the nfs service on different platforms
678 ########################################################
681 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
684 [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
692 service nfsserver start
695 service nfsserver stop > /dev/null 2>&1
698 set_proc "fs/nfsd/threads" 0
699 service nfsserver stop > /dev/null 2>&1
701 nfs_dump_some_threads
702 service nfsserver start
709 service nfslock start
717 set_proc "fs/nfsd/threads" 0
718 service nfs stop > /dev/null 2>&1
719 service nfslock stop > /dev/null 2>&1
721 nfs_dump_some_threads
722 service nfslock start
728 echo "Unknown platform. NFS is not supported with ctdb"
734 # Dump up to the configured number of nfsd thread backtraces.
735 nfs_dump_some_threads ()
737 [ -n "$CTDB_NFS_DUMP_STUCK_THREADS" ] || return 0
739 # Optimisation to avoid running an unnecessary pidof
740 [ $CTDB_NFS_DUMP_STUCK_THREADS -gt 0 ] || return 0
743 for _pid in $(pidof nfsd) ; do
744 [ $_count -le $CTDB_NFS_DUMP_STUCK_THREADS ] || break
746 # Do this first to avoid racing with thread exit
747 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
748 if [ -n "$_stack" ] ; then
749 echo "Stack trace for stuck nfsd thread [${_pid}]:"
751 _count=$(($_count + 1))
756 ########################################################
757 # start/stop the nfs lockmanager service on different platforms
758 ########################################################
759 startstop_nfslock() {
761 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
764 [ -x $CTDB_ETCDIR/init.d/nfslock ] && {
770 # for sles there is no service for lockmanager
771 # so we instead just shutdown/restart nfs
774 service nfsserver start
777 service nfsserver stop > /dev/null 2>&1
780 service nfsserver stop > /dev/null 2>&1
781 service nfsserver start
788 service nfslock start
791 service nfslock stop > /dev/null 2>&1
794 service nfslock stop > /dev/null 2>&1
795 service nfslock start
800 echo "Unknown platform. NFS locking is not supported with ctdb"
806 # Periodically update the statd database
811 _statd_update_trigger="$service_state_dir/update-trigger"
812 [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
814 _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
815 _current_time=$(date +"%s")
816 if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
817 touch "$_statd_update_trigger"
818 $CTDB_BASE/statd-callout updatelocal &
819 $CTDB_BASE/statd-callout updateremote &
829 _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
830 mkdir -p "${_lockfile%/*}" # dirname
831 [ -f "$_lockfile" ] || touch "$_lockfile"
834 # Note: use of return/exit/die() below only gets us out of the
835 # sub-shell, which is actually what we want. That is, the
836 # function should just return non-zero.
838 flock --timeout 30 0 || \
839 die "add_ip_to_iface: unable to get lock for ${_iface}"
841 # Ensure interface is up
842 ip link set "$_iface" up || \
843 die "Failed to bringup interface $_iface"
845 ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || \
846 die "Failed to add $_ip/$_maskbits on dev $_iface"
849 # Do nothing here - return above only gets us out of the subshell
850 # and doing anything here will affect the return code.
853 delete_ip_from_iface()
859 _lockfile="${CTDB_VARDIR}/state/interface_modify_${_iface}.flock"
860 mkdir -p "${_lockfile%/*}" # dirname
861 [ -f "$_lockfile" ] || touch "$_lockfile"
864 # Note: use of return/exit/die() below only gets us out of the
865 # sub-shell, which is actually what we want. That is, the
866 # function should just return non-zero.
868 flock --timeout 30 0 || \
869 die "delete_ip_from_iface: unable to get lock for ${_iface}"
871 _im="$_ip/$_maskbits" # shorthand for readability
873 # "ip addr del" will delete all secondary IPs if this is the
874 # primary. To work around this _very_ annoying behaviour we
875 # have to keep a record of the secondaries and re-add them
879 if ip addr list dev "$_iface" primary | grep -Fq "inet $_im " ; then
880 _secondaries=$(ip addr list dev "$_iface" secondary | \
881 awk '$1 == "inet" { print $2 }')
885 ip addr del "$_im" dev "$_iface" || {
886 echo "Failed to del $_ip on dev $_iface"
890 if [ -n "$_secondaries" ] ; then
891 for _i in $_secondaries; do
892 if ip addr list dev "$_iface" | grep -Fq "inet $_i" ; then
893 echo "Kept secondary $_i on dev $_iface"
895 echo "Re-adding secondary address $_i to dev $_iface"
896 ip addr add $_i brd + dev $_iface || {
897 echo "Failed to re-add address $_i to dev $_iface"
907 # Do nothing here - return above only gets us out of the subshell
908 # and doing anything here will affect the return code.
911 # If the given IP is hosted then print 2 items: maskbits and iface
916 ip addr show to "${_addr}/32" 2>/dev/null | \
917 awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }'
922 _addr="${1%/*}" # Remove optional maskbits
924 set -- $(ip_maskbits_iface $_addr)
925 if [ -n "$1" ] ; then
928 echo "Removing public address $_addr/$_maskbits from device $_iface"
929 delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
933 drop_all_public_ips ()
935 while read _ip _x ; do
937 done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
940 ########################################################
942 _ctdb_counter_common () {
943 _service_name="${1:-${service_name:-${script_name}}}"
944 _counter_file="$ctdb_fail_dir/$_service_name"
945 mkdir -p "${_counter_file%/*}" # dirname
947 ctdb_counter_init () {
948 _ctdb_counter_common "$1"
952 ctdb_counter_incr () {
953 _ctdb_counter_common "$1"
956 echo -n 1 >> "$_counter_file"
958 ctdb_check_counter () {
959 _msg="${1:-error}" # "error" - anything else is silent on fail
960 _op="${2:--ge}" # an integer operator supported by test
961 _limit="${3:-${service_fail_limit}}"
963 _ctdb_counter_common "$1"
966 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
968 if [ "$_op" != "%" ] ; then
969 if [ $_size $_op $_limit ] ; then
973 if [ $(($_size $_op $_limit)) -eq 0 ] ; then
978 if [ "$_msg" = "error" ] ; then
979 echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
987 ########################################################
989 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
990 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
992 ctdb_setup_service_state_dir ()
994 service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
995 mkdir -p "$service_state_dir" || {
996 echo "Error creating state dir \"$service_state_dir\""
1001 ########################################################
1002 # Managed status history, for auto-start/stop
1004 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1006 _ctdb_managed_common ()
1008 _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1011 ctdb_service_managed ()
1013 _ctdb_managed_common
1014 mkdir -p "$ctdb_managed_dir"
1015 touch "$_ctdb_managed_file"
1018 ctdb_service_unmanaged ()
1020 _ctdb_managed_common
1021 rm -f "$_ctdb_managed_file"
1024 is_ctdb_previously_managed_service ()
1026 _ctdb_managed_common
1027 [ -f "$_ctdb_managed_file" ]
1030 ########################################################
1031 # Check and set status
1035 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1040 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1041 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1043 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1044 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1053 d="$ctdb_status_dir/$script_name"
1060 for i in "banned" "unhealthy" ; do
1067 ##################################################################
1068 # Reconfigure a service on demand
1070 _ctdb_service_reconfigure_common ()
1072 _d="$ctdb_status_dir/${service_name}"
1074 _ctdb_service_reconfigure_flag="$_d/reconfigure"
1077 ctdb_service_needs_reconfigure ()
1079 _ctdb_service_reconfigure_common
1080 [ -e "$_ctdb_service_reconfigure_flag" ]
1083 ctdb_service_set_reconfigure ()
1085 _ctdb_service_reconfigure_common
1086 >"$_ctdb_service_reconfigure_flag"
1089 ctdb_service_unset_reconfigure ()
1091 _ctdb_service_reconfigure_common
1092 rm -f "$_ctdb_service_reconfigure_flag"
1095 ctdb_service_reconfigure ()
1097 echo "Reconfiguring service \"${service_name}\"..."
1098 ctdb_service_unset_reconfigure
1099 service_reconfigure || return $?
1103 # Default service_reconfigure() function does nothing.
1104 service_reconfigure ()
1109 ctdb_reconfigure_take_lock ()
1111 _ctdb_service_reconfigure_common
1112 _lock="${_d}/reconfigure_lock"
1113 mkdir -p "${_lock%/*}" # dirname
1118 # This is overkill but will work if we need to extend this to
1119 # allow certain events to run multiple times in parallel
1120 # (e.g. takeip) and write multiple PIDs to the file.
1122 if [ -n "$_locker_event" ] ; then
1123 while read _pid ; do
1124 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1125 kill -0 "$_pid" 2>/dev/null ; then
1131 printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1136 ctdb_reconfigure_release_lock ()
1138 _ctdb_service_reconfigure_common
1139 _lock="${_d}/reconfigure_lock"
1144 ctdb_replay_monitor_status ()
1146 echo "Replaying previous status for this script due to reconfigure..."
1147 # Leading colon (':') is missing in some versions...
1148 _out=$(ctdb scriptstatus -Y | grep -E "^:?monitor:${script_name}:")
1149 # Output looks like this:
1150 # :monitor:60.nfs:1:ERROR:1314764004.030861:1314764004.035514:foo bar:
1151 # This is the cheapest way of getting fields in the middle.
1152 set -- $(IFS=":" ; echo $_out)
1155 # The error output field can include colons so we'll try to
1156 # preserve them. The weak checking at the beginning tries to make
1157 # this work for both broken (no leading ':') and fixed output.
1159 _err_out="${_out#*monitor:${script_name}:*:*:*:*:}"
1161 OK) : ;; # Do nothing special.
1163 # Recast this as an error, since we can't exit with the
1164 # correct negative number.
1166 _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1169 # Recast this as an OK, since we can't exit with the
1170 # correct negative number.
1172 _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1174 *) : ;; # Must be ERROR, do nothing special.
1176 if [ -n "$_err_out" ] ; then
1182 ctdb_service_check_reconfigure ()
1186 # We only care about some events in this function. For others we
1188 case "$event_name" in
1189 monitor|ipreallocated|reconfigure) : ;;
1193 if ctdb_reconfigure_take_lock ; then
1194 # No events covered by this function are running, so proceed
1196 case "$event_name" in
1198 (ctdb_service_reconfigure)
1202 if ctdb_service_needs_reconfigure ; then
1203 ctdb_service_reconfigure
1208 ctdb_reconfigure_release_lock
1210 # Somebody else is running an event we don't want to collide
1211 # with. We proceed with caution.
1212 case "$event_name" in
1214 # Tell whoever called us to retry.
1218 # Defer any scheduled reconfigure and just run the
1219 # rest of the ipreallocated event, as per the
1220 # eventscript. There's an assumption here that the
1221 # event doesn't depend on any scheduled reconfigure.
1222 # This is true in the current code.
1226 # There is most likely a reconfigure in progress so
1227 # the service is possibly unstable. As above, we
1228 # defer any scheduled reconfigured. We also replay
1229 # the previous monitor status since that's the best
1230 # information we have.
1231 ctdb_replay_monitor_status
1237 ##################################################################
1238 # Does CTDB manage this service? - and associated auto-start/stop
1240 ctdb_compat_managed_service ()
1242 if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1243 CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1247 is_ctdb_managed_service ()
1251 # $t is used just for readability and to allow better accurate
1252 # matching via leading/trailing spaces
1253 t=" $CTDB_MANAGED_SERVICES "
1255 # Return 0 if "<space>$service_name<space>" appears in $t
1256 if [ "${t#* ${service_name} }" != "${t}" ] ; then
1260 # If above didn't match then update $CTDB_MANAGED_SERVICES for
1261 # backward compatibility and try again.
1262 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
1263 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
1264 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
1265 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "apache2"
1266 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
1267 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
1268 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
1269 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
1270 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
1272 t=" $CTDB_MANAGED_SERVICES "
1274 # Return 0 if "<space>$service_name<space>" appears in $t
1275 [ "${t#* ${service_name} }" != "${t}" ]
1278 ctdb_start_stop_service ()
1282 # Allow service-start/service-stop pseudo-events to start/stop
1283 # services when we're not auto-starting/stopping and we're not
1285 case "$event_name" in
1287 if is_ctdb_managed_service ; then
1288 die 'service-start event not permitted when service is managed'
1290 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1291 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1297 if is_ctdb_managed_service ; then
1298 die 'service-stop event not permitted when service is managed'
1300 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1301 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1308 # Do nothing unless configured to...
1309 [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1311 [ "$event_name" = "monitor" ] || return 0
1313 if is_ctdb_managed_service ; then
1314 if ! is_ctdb_previously_managed_service ; then
1315 echo "Starting service \"$service_name\" - now managed"
1316 background_with_logging ctdb_service_start
1320 if is_ctdb_previously_managed_service ; then
1321 echo "Stopping service \"$service_name\" - no longer managed"
1322 background_with_logging ctdb_service_stop
1328 ctdb_service_start ()
1330 # The service is marked managed if we've ever tried to start it.
1331 ctdb_service_managed
1333 service_start || return $?
1339 ctdb_service_stop ()
1341 ctdb_service_unmanaged
1345 # Default service_start() and service_stop() functions.
1347 # These may be overridden in an eventscript. When overriding, the
1348 # following convention must be followed. If these functions are
1349 # called with no arguments then they may use internal logic to
1350 # determine whether the service is managed and, therefore, whether
1351 # they should take any action. However, if the service name is
1352 # specified as an argument then an attempt must be made to start or
1353 # stop the service. This is because the auto-start/stop code calls
1354 # them with the service name as an argument.
1357 service "$service_name" start
1362 service "$service_name" stop
1365 ##################################################################
1367 ctdb_standard_event_handler ()
1382 # iptables doesn't like being re-entered, so flock-wrap it.
1385 flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1388 # AIX (and perhaps others?) doesn't have mktemp
1389 if ! which mktemp >/dev/null 2>&1 ; then
1393 if [ "$1" = "-d" ] ; then
1397 _d="${TMPDIR:-/tmp}"
1398 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1400 sed -e 's@\(..........\).*@\1@')
1401 _t="${_d}/tmp.${_hex10}"
1414 ########################################################
1416 ########################################################
1422 tickledir="$CTDB_VARDIR/state/tickles"
1423 mkdir -p "$tickledir"
1426 _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1428 # What public IPs do I hold?
1429 _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
1431 # IPs as a regexp choice
1432 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1434 # Record connections to our public IPs in a temporary file
1435 _my_connections="${tickledir}/${_port}.connections"
1436 rm -f "$_my_connections"
1438 awk -v destpat="^${_ipschoice}:${_port}\$" \
1439 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1440 sort >"$_my_connections"
1442 # Record our current tickles in a temporary file
1443 _my_tickles="${tickledir}/${_port}.tickles"
1444 rm -f "$_my_tickles"
1445 for _i in $_ips ; do
1446 ctdb -Y gettickles $_i $_port |
1447 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1449 sort >"$_my_tickles"
1451 # Add tickles for connections that we haven't already got tickles for
1452 comm -23 "$_my_connections" "$_my_tickles" |
1453 while read _src _dst ; do
1454 ctdb addtickle $_src $_dst
1457 # Remove tickles for connections that are no longer there
1458 comm -13 "$_my_connections" "$_my_tickles" |
1459 while read _src _dst ; do
1460 ctdb deltickle $_src $_dst
1463 rm -f "$_my_connections" "$_my_tickles"
1466 ########################################################
1467 # load a site local config file
1468 ########################################################
1470 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1474 [ -x $CTDB_BASE/rc.local ] && {
1475 . $CTDB_BASE/rc.local
1478 [ -d $CTDB_BASE/rc.local.d ] && {
1479 for i in $CTDB_BASE/rc.local.d/* ; do
1480 [ -x "$i" ] && . "$i"
1484 script_name="${0##*/}" # basename
1485 service_fail_limit=1