| blob | 77c3ffbac36588ffba6a720382c3562924caa24b |
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6 if [ -d "/var/lib/ctdb" ] ; then
7 export CTDB_VARDIR="/var/lib/ctdb"
8 else
9 export CTDB_VARDIR="/var/ctdb"
10 fi
11 }
12 [ -z "$CTDB_ETCDIR" ] && {
13 export CTDB_ETCDIR="/etc"
14 }
16 #######################################
17 # pull in a system config file, if any
18 _loadconfig() {
20 if [ -z "$1" ] ; then
21 foo="${service_config:-${service_name}}"
22 if [ -n "$foo" ] ; then
23 loadconfig "$foo"
24 return
25 fi
26 fi
28 if [ "$1" != "ctdb" ] ; then
29 loadconfig "ctdb"
30 fi
32 if [ -z "$1" ] ; then
33 return
34 fi
36 if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37 . $CTDB_ETCDIR/sysconfig/$1
38 elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39 . $CTDB_ETCDIR/default/$1
40 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41 . $CTDB_BASE/sysconfig/$1
42 fi
44 if [ "$1" = "ctdb" ] ; then
45 _config="${CTDB_BASE}/ctdbd.conf"
46 if [ -r "$_config" ] ; then
47 . "$_config"
48 fi
49 fi
50 }
52 loadconfig () {
53 _loadconfig "$@"
54 }
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
59 # configuration file.
60 debug ()
61 {
62 if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63 # If there are arguments then echo them. Otherwise expect to
64 # use stdin, which allows us to pass lots of debug using a
65 # here document.
66 if [ -n "$1" ] ; then
67 echo "DEBUG: $*"
68 elif ! tty -s ; then
69 sed -e 's@^@DEBUG: @'
70 fi
71 fi
72 }
74 die ()
75 {
76 _msg="$1"
77 _rc="${2:-1}"
79 echo "$_msg"
80 exit $_rc
81 }
83 # Log given message or stdin to either syslog or a CTDB log file
84 # $1 is the tag passed to logger if syslog is in use.
85 script_log ()
86 {
87 _tag="$1" ; shift
89 case "$CTDB_LOGGING" in
90 file:*|"")
91 if [ -n "$CTDB_LOGGING" ] ; then
92 _file="${CTDB_LOGGING#file:}"
93 else
94 _file="/var/log/log.ctdb"
95 fi
96 {
97 if [ -n "$*" ] ; then
98 echo "$*"
99 else
100 cat
101 fi
102 } >>"$_file"
103 ;;
104 *)
105 # Handle all syslog:* variants here too. There's no tool to do
106 # the lossy things, so just use logger.
107 logger -t "ctdbd: ${_tag}" $*
108 ;;
109 esac
110 }
112 # When things are run in the background in an eventscript then logging
113 # output might get lost. This is the "solution". :-)
114 background_with_logging ()
115 {
116 (
117 "$@" 2>&1 </dev/null |
118 script_log "${script_name}&"
119 )&
121 return 0
122 }
124 ##############################################################
125 # check number of args for different events
126 ctdb_check_args ()
127 {
128 case "$1" in
129 takeip|releaseip)
130 if [ $# != 4 ]; then
131 echo "ERROR: must supply interface, IP and maskbits"
132 exit 1
133 fi
134 ;;
135 updateip)
136 if [ $# != 5 ]; then
137 echo "ERROR: must supply old interface, new interface, IP and maskbits"
138 exit 1
139 fi
140 ;;
141 esac
142 }
144 ##############################################################
145 # determine on what type of system (init style) we are running
146 detect_init_style()
147 {
148 # only do detection if not already set:
149 [ -z "$CTDB_INIT_STYLE" ] || return
151 if [ -x /sbin/startproc ]; then
152 CTDB_INIT_STYLE="suse"
153 elif [ -x /sbin/start-stop-daemon ]; then
154 CTDB_INIT_STYLE="debian"
155 else
156 CTDB_INIT_STYLE="redhat"
157 fi
158 }
160 ######################################################
161 # simulate /sbin/service on platforms that don't have it
162 # _service() makes it easier to hook the service() function for
163 # testing.
164 _service ()
165 {
166 _service_name="$1"
167 _op="$2"
169 # do nothing, when no service was specified
170 [ -z "$_service_name" ] && return
172 if [ -x /sbin/service ]; then
173 $_nice /sbin/service "$_service_name" "$_op"
174 elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
175 $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
176 elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
177 $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
178 fi
179 }
181 service()
182 {
183 _nice=""
184 _service "$@"
185 }
187 ######################################################
188 # simulate /sbin/service (niced) on platforms that don't have it
189 nice_service()
190 {
191 _nice="nice"
192 _service "$@"
193 }
195 ######################################################
196 # wrapper around /proc/ settings to allow them to be hooked
197 # for testing
198 # 1st arg is relative path under /proc/, 2nd arg is value to set
199 set_proc ()
200 {
201 echo "$2" >"/proc/$1"
202 }
204 ######################################################
205 # wrapper around getting file contents from /proc/ to allow
206 # this to be hooked for testing
207 # 1st arg is relative path under /proc/
208 get_proc ()
209 {
210 cat "/proc/$1"
211 }
213 ######################################################
214 # Print up to $_max kernel stack traces for processes named $_program
215 program_stack_traces ()
216 {
217 _prog="$1"
218 _max="${2:-1}"
220 _count=1
221 for _pid in $(pidof "$_prog") ; do
222 [ $_count -le $_max ] || break
224 # Do this first to avoid racing with process exit
225 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
226 if [ -n "$_stack" ] ; then
227 echo "Stack trace for ${_prog}[${_pid}]:"
228 echo "$_stack"
229 _count=$(($_count + 1))
230 fi
231 done
232 }
234 ######################################################
235 # Check that an RPC service is healthy -
236 # this includes allowing a certain number of failures
237 # before marking the NFS service unhealthy.
238 #
239 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
240 #
241 # each triple is a set of 3 arguments: an operator, a
242 # fail count limit and an action string.
243 #
244 # For example:
245 #
246 # nfs_check_rpc_service "lockd" \
247 # -ge 15 "verbose restart unhealthy" \
248 # -eq 10 "restart:bs"
249 #
250 # says that if lockd is down for 15 iterations then do
251 # a verbose restart of lockd and mark the node unhealthy.
252 # Before this, after 10 iterations of failure, the
253 # service is restarted silently in the background.
254 # Order is important: the number of failures need to be
255 # specified in reverse order because processing stops
256 # after the first condition that is true.
257 ######################################################
258 nfs_check_rpc_service ()
259 {
260 _prog_name="$1" ; shift
262 if _nfs_check_rpc_common "$_prog_name" ; then
263 return
264 fi
266 while [ -n "$3" ] ; do
267 if _nfs_check_rpc_action "$1" "$2" "$3" ; then
268 break
269 fi
270 shift 3
271 done
272 }
274 # The new way of doing things...
275 nfs_check_rpc_services ()
276 {
277 # Files must end with .check - avoids editor backups, RPM fu, ...
278 for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
279 _t="${_f%.check}"
280 _prog_name="${_t##*/[0-9][0-9].}"
282 if _nfs_check_rpc_common "$_prog_name" ; then
283 # This RPC service is up, check next service...
284 continue
285 fi
287 # Check each line in the file in turn until one of the limit
288 # checks is hit...
289 while read _cmp _lim _rest ; do
290 # Skip comments
291 case "$_cmp" in
292 \#*) continue ;;
293 esac
295 if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
296 # Limit was hit on this line, no further checking...
297 break
298 fi
299 done <"$_f"
300 done
301 }
303 _nfs_check_rpc_common ()
304 {
305 _prog_name="$1"
307 # Some platforms don't have separate programs for all services.
308 case "$_prog_name" in
309 statd)
310 which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
311 esac
313 case "$_prog_name" in
314 nfsd)
315 _rpc_prog=nfs
316 _version=3
317 ;;
318 mountd)
319 _rpc_prog=mountd
320 _version=1
321 ;;
322 rquotad)
323 _rpc_prog=rquotad
324 _version=1
325 ;;
326 lockd)
327 _rpc_prog=nlockmgr
328 _version=4
329 ;;
330 statd)
331 _rpc_prog=status
332 _version=1
333 ;;
334 *)
335 echo "Internal error: unknown RPC program \"$_prog_name\"."
336 exit 1
337 esac
339 _service_name="nfs_${_prog_name}"
341 if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
342 ctdb_counter_init "$_service_name"
343 return 0
344 fi
346 ctdb_counter_incr "$_service_name"
348 return 1
349 }
351 _nfs_check_rpc_action ()
352 {
353 _cmp="$1"
354 _limit="$2"
355 _actions="$3"
357 if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
358 return 1
359 fi
361 for _action in $_actions ; do
362 case "$_action" in
363 verbose)
364 echo "$ctdb_check_rpc_out"
365 ;;
366 restart)
367 _nfs_restart_rpc_service "$_prog_name"
368 ;;
369 restart:b)
370 _nfs_restart_rpc_service "$_prog_name" true
371 ;;
372 unhealthy)
373 exit 1
374 ;;
375 *)
376 echo "Internal error: unknown action \"$_action\"."
377 exit 1
378 esac
379 done
381 return 0
382 }
384 _nfs_restart_rpc_service ()
385 {
386 _prog_name="$1"
387 _background="${2:-false}"
389 if $_background ; then
390 _maybe_background="background_with_logging"
391 else
392 _maybe_background=""
393 fi
395 _p="rpc.${_prog_name}"
397 case "$_prog_name" in
398 nfsd)
399 echo "Trying to restart NFS service"
400 $_maybe_background startstop_nfs restart
401 ;;
402 mountd)
403 echo "Trying to restart $_prog_name [${_p}]"
404 killall -q -9 "$_p"
405 nfs_dump_some_threads "$_p"
406 $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
407 ;;
408 rquotad)
409 echo "Trying to restart $_prog_name [${_p}]"
410 killall -q -9 "$_p"
411 nfs_dump_some_threads "$_p"
412 $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
413 ;;
414 lockd)
415 echo "Trying to restart lock manager service"
416 $_maybe_background startstop_nfslock restart
417 ;;
418 statd)
419 echo "Trying to restart $_prog_name [${_p}]"
420 killall -q -9 "$_p"
421 nfs_dump_some_threads "$_p"
422 $_maybe_background $_p \
423 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
424 ${STATD_PORT:+-p} $STATD_PORT \
425 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
426 ;;
427 *)
428 echo "Internal error: unknown RPC program \"$_prog_name\"."
429 exit 1
430 esac
431 }
433 ######################################################
434 # check that a rpc server is registered with portmap
435 # and responding to requests
436 # usage: ctdb_check_rpc SERVICE_NAME VERSION
437 ######################################################
438 ctdb_check_rpc ()
439 {
440 progname="$1"
441 version="$2"
443 _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
445 if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
446 ctdb_check_rpc_out="ERROR: $progname failed RPC check:
447 $ctdb_check_rpc_out"
448 echo "$ctdb_check_rpc_out"
449 return 1
450 fi
451 }
453 ######################################################
454 # Ensure $service_name is set
455 assert_service_name ()
456 {
457 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
458 }
460 ######################################################
461 # check a set of directories is available
462 # return 1 on a missing directory
463 # directories are read from stdin
464 ######################################################
465 ctdb_check_directories_probe()
466 {
467 while IFS="" read d ; do
468 case "$d" in
469 *%*)
470 continue
471 ;;
472 *)
473 [ -d "${d}/." ] || return 1
474 esac
475 done
476 }
478 ######################################################
479 # check a set of directories is available
480 # directories are read from stdin
481 ######################################################
482 ctdb_check_directories()
483 {
484 ctdb_check_directories_probe || {
485 echo "ERROR: $service_name directory \"$d\" not available"
486 exit 1
487 }
488 }
490 ######################################################
491 # check a set of tcp ports
492 # usage: ctdb_check_tcp_ports <ports...>
493 ######################################################
495 # This flag file is created when a service is initially started. It
496 # is deleted the first time TCP port checks for that service succeed.
497 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
498 # message if a port check fails.
499 _ctdb_check_tcp_common ()
500 {
501 assert_service_name
502 _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
503 }
505 ctdb_check_tcp_init ()
506 {
507 _ctdb_check_tcp_common
508 mkdir -p "${_ctdb_service_started_file%/*}" # dirname
509 touch "$_ctdb_service_started_file"
510 }
512 # Check whether something is listening on all of the given TCP ports
513 # using the "ctdb checktcpport" command.
514 ctdb_check_tcp_ports()
515 {
516 if [ -z "$1" ] ; then
517 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
518 exit 1
519 fi
521 for _p ; do # process each function argument (port)
522 _cmd="ctdb checktcpport $_p"
523 _out=$($_cmd 2>&1)
524 _ret=$?
525 case "$_ret" in
526 0)
527 _ctdb_check_tcp_common
528 if [ ! -f "$_ctdb_service_started_file" ] ; then
529 echo "ERROR: $service_name tcp port $_p is not responding"
530 debug "\"ctdb checktcpport $_p\" was able to bind to port"
531 else
532 echo "INFO: $service_name tcp port $_p is not responding"
533 fi
535 return 1
536 ;;
537 98)
538 # Couldn't bind, something already listening, next port...
539 continue
540 ;;
541 *)
542 echo "ERROR: unexpected error running \"ctdb checktcpport\""
543 debug <<EOF
544 ctdb checktcpport (exited with $_ret) with output:
545 $_out"
546 EOF
547 return $_ret
548 esac
549 done
551 # All ports listening
552 _ctdb_check_tcp_common
553 rm -f "$_ctdb_service_started_file"
554 return 0
555 }
557 ######################################################
558 # check a unix socket
559 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
560 ######################################################
561 ctdb_check_unix_socket() {
562 socket_path="$1"
563 [ -z "$socket_path" ] && return
565 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
566 echo "ERROR: $service_name socket $socket_path not found"
567 return 1
568 fi
569 }
571 ######################################################
572 # check a command returns zero status
573 # usage: ctdb_check_command <command>
574 ######################################################
575 ctdb_check_command ()
576 {
577 _out=$("$@" 2>&1) || {
578 echo "ERROR: $* returned error"
579 echo "$_out" | debug
580 exit 1
581 }
582 }
584 ################################################
585 # kill off any TCP connections with the given IP
586 ################################################
587 kill_tcp_connections ()
588 {
589 _ip="$1"
591 _oneway=false
592 if [ "$2" = "oneway" ] ; then
593 _oneway=true
594 fi
596 get_tcp_connections_for_ip "$_ip" | {
597 _killcount=0
598 _connections=""
599 _nl="
600 "
601 while read _dst _src; do
602 _destport="${_dst##*:}"
603 __oneway=$_oneway
604 case $_destport in
605 # we only do one-way killtcp for CIFS
606 139|445) __oneway=true ;;
607 esac
609 echo "Killing TCP connection $_src $_dst"
610 _connections="${_connections}${_nl}${_src} ${_dst}"
611 if ! $__oneway ; then
612 _connections="${_connections}${_nl}${_dst} ${_src}"
613 fi
615 _killcount=$(($_killcount + 1))
616 done
618 if [ $_killcount -eq 0 ] ; then
619 return
620 fi
622 echo "$_connections" | ctdb killtcp || {
623 echo "Failed to send killtcp control"
624 return
625 }
627 _count=0
628 while : ; do
629 _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
631 if [ $_remaining -eq 0 ] ; then
632 echo "Killed $_killcount TCP connections to released IP $_ip"
633 return
634 fi
636 _count=$(($_count + 1))
637 if [ $_count -gt 3 ] ; then
638 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
639 return
640 fi
642 echo "Waiting for $_remaining connections to be killed for IP $_ip"
643 sleep 1
644 done
645 }
646 }
648 ##################################################################
649 # kill off the local end for any TCP connections with the given IP
650 ##################################################################
651 kill_tcp_connections_local_only ()
652 {
653 kill_tcp_connections "$1" "oneway"
654 }
656 ##################################################################
657 # tickle any TCP connections with the given IP
658 ##################################################################
659 tickle_tcp_connections ()
660 {
661 _ip="$1"
663 get_tcp_connections_for_ip "$_ip" |
664 {
665 _failed=false
667 while read dest src; do
668 echo "Tickle TCP connection $src $dest"
669 ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
670 echo "Tickle TCP connection $dest $src"
671 ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
672 done
674 if $_failed ; then
675 echo "Failed to send tickle control"
676 fi
677 }
678 }
680 get_tcp_connections_for_ip ()
681 {
682 _ip="$1"
684 netstat -tn | awk -v ip=$_ip \
685 'index($1, "tcp") == 1 && \
686 (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
687 && $6 == "ESTABLISHED" \
688 {print $4" "$5}'
689 }
691 ########################################################
692 # start/stop the Ganesha nfs service
693 ########################################################
694 startstop_ganesha()
695 {
696 _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
697 case "$1" in
698 start)
699 service "$_service_name" start
700 ;;
701 stop)
702 service "$_service_name" stop
703 ;;
704 restart)
705 service "$_service_name" stop
706 nfs_dump_some_threads "rpc.statd"
707 service "$_service_name" start
708 ;;
709 esac
710 }
712 ########################################################
713 # start/stop the nfs service on different platforms
714 ########################################################
715 startstop_nfs() {
716 PLATFORM="unknown"
717 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
718 PLATFORM="sles"
719 }
720 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
721 -r /usr/lib/systemd/system/nfs-lock.service ] && {
722 PLATFORM="rhel"
723 }
725 case $PLATFORM in
726 sles)
727 case $1 in
728 start)
729 service nfsserver start
730 ;;
731 stop)
732 service nfsserver stop > /dev/null 2>&1
733 ;;
734 restart)
735 set_proc "fs/nfsd/threads" 0
736 service nfsserver stop > /dev/null 2>&1
737 pkill -9 nfsd
738 nfs_dump_some_threads
739 service nfsserver start
740 ;;
741 esac
742 ;;
743 rhel)
744 case $1 in
745 start)
746 service nfslock start
747 service nfs start
748 ;;
749 stop)
750 service nfs stop
751 service nfslock stop
752 ;;
753 restart)
754 set_proc "fs/nfsd/threads" 0
755 service nfs stop > /dev/null 2>&1
756 service nfslock stop > /dev/null 2>&1
757 pkill -9 nfsd
758 nfs_dump_some_threads
759 service nfslock start
760 service nfs start
761 ;;
762 esac
763 ;;
764 *)
765 echo "Unknown platform. NFS is not supported with ctdb"
766 exit 1
767 ;;
768 esac
769 }
771 # Dump up to the configured number of nfsd thread backtraces.
772 nfs_dump_some_threads ()
773 {
774 _prog="${1:-nfsd}"
776 _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
777 [ $_num -gt 0 ] || return 0
779 program_stack_traces "$_prog" $_num
780 }
782 ########################################################
783 # start/stop the nfs lockmanager service on different platforms
784 ########################################################
785 startstop_nfslock() {
786 PLATFORM="unknown"
787 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
788 PLATFORM="sles"
789 }
790 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
791 -r /usr/lib/systemd/system/nfs-lock.service ] && {
792 PLATFORM="rhel"
793 }
795 case $PLATFORM in
796 sles)
797 # for sles there is no service for lockmanager
798 # so we instead just shutdown/restart nfs
799 case $1 in
800 start)
801 service nfsserver start
802 ;;
803 stop)
804 service nfsserver stop > /dev/null 2>&1
805 ;;
806 restart)
807 service nfsserver stop > /dev/null 2>&1
808 service nfsserver start
809 ;;
810 esac
811 ;;
812 rhel)
813 case $1 in
814 start)
815 service nfslock start
816 ;;
817 stop)
818 service nfslock stop > /dev/null 2>&1
819 ;;
820 restart)
821 service nfslock stop > /dev/null 2>&1
822 service nfslock start
823 ;;
824 esac
825 ;;
826 *)
827 echo "Unknown platform. NFS locking is not supported with ctdb"
828 exit 1
829 ;;
830 esac
831 }
833 # Periodically update the statd database
834 nfs_statd_update ()
835 {
836 _update_period="$1"
838 _statd_update_trigger="$service_state_dir/update-trigger"
839 [ -f "$_statd_update_trigger" ] || touch "$_statd_update_trigger"
841 _last_update=$(stat --printf="%Y" "$_statd_update_trigger")
842 _current_time=$(date +"%s")
843 if [ $(( $_current_time - $_last_update)) -ge $_update_period ] ; then
844 touch "$_statd_update_trigger"
845 $CTDB_BASE/statd-callout updatelocal &
846 $CTDB_BASE/statd-callout updateremote &
847 fi
848 }
850 ########################################################
852 add_ip_to_iface ()
853 {
854 _iface=$1
855 _ip=$2
856 _maskbits=$3
858 # Ensure interface is up
859 ip link set "$_iface" up || \
860 die "Failed to bringup interface $_iface"
862 ip addr add "$_ip/$_maskbits" brd + dev "$_iface" || {
863 echo "Failed to add $_ip/$_maskbits on dev $_iface"
864 return 1
865 }
866 }
868 delete_ip_from_iface()
869 {
870 _iface=$1
871 _ip=$2
872 _maskbits=$3
874 # This could be set globally for all interfaces but it is probably
875 # better to avoid surprises, so limit it the interfaces where CTDB
876 # has public IP addresses. There isn't anywhere else convenient
877 # to do this so just set it each time. This is much cheaper than
878 # remembering and re-adding secondaries.
879 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
881 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
882 echo "Failed to del $_ip on dev $_iface"
883 return 1
884 }
885 }
887 # If the given IP is hosted then print 2 items: maskbits and iface
888 ip_maskbits_iface ()
889 {
890 _addr="$1"
892 ip addr show to "${_addr}/32" 2>/dev/null | \
893 awk '$1 == "inet" { print gensub(".*/", "", 1, $2), $NF }'
894 }
896 drop_ip ()
897 {
898 _addr="${1%/*}" # Remove optional maskbits
900 set -- $(ip_maskbits_iface $_addr)
901 if [ -n "$1" ] ; then
902 _maskbits="$1"
903 _iface="$2"
904 echo "Removing public address $_addr/$_maskbits from device $_iface"
905 delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
906 fi
907 }
909 drop_all_public_ips ()
910 {
911 while read _ip _x ; do
912 drop_ip "$_ip"
913 done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
914 }
916 ########################################################
917 # Simple counters
918 _ctdb_counter_common () {
919 _service_name="${1:-${service_name:-${script_name}}}"
920 _counter_file="$ctdb_fail_dir/$_service_name"
921 mkdir -p "${_counter_file%/*}" # dirname
922 }
923 ctdb_counter_init () {
924 _ctdb_counter_common "$1"
926 >"$_counter_file"
927 }
928 ctdb_counter_incr () {
929 _ctdb_counter_common "$1"
931 # unary counting!
932 echo -n 1 >> "$_counter_file"
933 }
934 ctdb_check_counter () {
935 _msg="${1:-error}" # "error" - anything else is silent on fail
936 _op="${2:--ge}" # an integer operator supported by test
937 _limit="${3:-${service_fail_limit}}"
938 shift 3
939 _ctdb_counter_common "$1"
941 # unary counting!
942 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
943 _hit=false
944 if [ "$_op" != "%" ] ; then
945 if [ $_size $_op $_limit ] ; then
946 _hit=true
947 fi
948 else
949 if [ $(($_size $_op $_limit)) -eq 0 ] ; then
950 _hit=true
951 fi
952 fi
953 if $_hit ; then
954 if [ "$_msg" = "error" ] ; then
955 echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
956 exit 1
957 else
958 return 1
959 fi
960 fi
961 }
963 ########################################################
965 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
966 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
968 ctdb_setup_service_state_dir ()
969 {
970 service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
971 mkdir -p "$service_state_dir" || {
972 echo "Error creating state dir \"$service_state_dir\""
973 exit 1
974 }
975 }
977 ########################################################
978 # Managed status history, for auto-start/stop
980 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
982 _ctdb_managed_common ()
983 {
984 _ctdb_managed_file="$ctdb_managed_dir/$service_name"
985 }
987 ctdb_service_managed ()
988 {
989 _ctdb_managed_common
990 mkdir -p "$ctdb_managed_dir"
991 touch "$_ctdb_managed_file"
992 }
994 ctdb_service_unmanaged ()
995 {
996 _ctdb_managed_common
997 rm -f "$_ctdb_managed_file"
998 }
1000 is_ctdb_previously_managed_service ()
1001 {
1002 _ctdb_managed_common
1003 [ -f "$_ctdb_managed_file" ]
1004 }
1006 ########################################################
1007 # Check and set status
1009 log_status_cat ()
1010 {
1011 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1012 }
1014 ctdb_checkstatus ()
1015 {
1016 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1017 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1018 return 1
1019 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1020 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1021 return 2
1022 else
1023 return 0
1024 fi
1025 }
1027 ctdb_setstatus ()
1028 {
1029 d="$ctdb_status_dir/$script_name"
1030 case "$1" in
1031 unhealthy|banned)
1032 mkdir -p "$d"
1033 cat "$2" >"$d/$1"
1034 ;;
1035 *)
1036 for i in "banned" "unhealthy" ; do
1037 rm -f "$d/$i"
1038 done
1039 ;;
1040 esac
1041 }
1043 ##################################################################
1044 # Reconfigure a service on demand
1046 _ctdb_service_reconfigure_common ()
1047 {
1048 _d="$ctdb_status_dir/${service_name}"
1049 mkdir -p "$_d"
1050 _ctdb_service_reconfigure_flag="$_d/reconfigure"
1051 }
1053 ctdb_service_needs_reconfigure ()
1054 {
1055 _ctdb_service_reconfigure_common
1056 [ -e "$_ctdb_service_reconfigure_flag" ]
1057 }
1059 ctdb_service_set_reconfigure ()
1060 {
1061 _ctdb_service_reconfigure_common
1062 >"$_ctdb_service_reconfigure_flag"
1063 }
1065 ctdb_service_unset_reconfigure ()
1066 {
1067 _ctdb_service_reconfigure_common
1068 rm -f "$_ctdb_service_reconfigure_flag"
1069 }
1071 ctdb_service_reconfigure ()
1072 {
1073 echo "Reconfiguring service \"${service_name}\"..."
1074 ctdb_service_unset_reconfigure
1075 service_reconfigure || return $?
1076 ctdb_counter_init
1077 }
1079 # Default service_reconfigure() function does nothing.
1080 service_reconfigure ()
1081 {
1082 :
1083 }
1085 ctdb_reconfigure_take_lock ()
1086 {
1087 _ctdb_service_reconfigure_common
1088 _lock="${_d}/reconfigure_lock"
1089 mkdir -p "${_lock%/*}" # dirname
1090 touch "$_lock"
1092 (
1093 flock 0
1094 # This is overkill but will work if we need to extend this to
1095 # allow certain events to run multiple times in parallel
1096 # (e.g. takeip) and write multiple PIDs to the file.
1097 read _locker_event
1098 if [ -n "$_locker_event" ] ; then
1099 while read _pid ; do
1100 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1101 kill -0 "$_pid" 2>/dev/null ; then
1102 exit 1
1103 fi
1104 done
1105 fi
1107 printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1108 exit 0
1109 ) <"$_lock"
1110 }
1112 ctdb_reconfigure_release_lock ()
1113 {
1114 _ctdb_service_reconfigure_common
1115 _lock="${_d}/reconfigure_lock"
1117 rm -f "$_lock"
1118 }
1120 ctdb_replay_monitor_status ()
1121 {
1122 echo "Replaying previous status for this script due to reconfigure..."
1123 # Leading colon (':') is missing in some versions...
1124 _out=$(ctdb scriptstatus -Y | grep -E "^:?monitor:${script_name}:")
1125 # Output looks like this:
1126 # :monitor:60.nfs:1:ERROR:1314764004.030861:1314764004.035514:foo bar:
1127 # This is the cheapest way of getting fields in the middle.
1128 set -- $(IFS=":" ; echo $_out)
1129 _code="$3"
1130 _status="$4"
1131 # The error output field can include colons so we'll try to
1132 # preserve them. The weak checking at the beginning tries to make
1133 # this work for both broken (no leading ':') and fixed output.
1134 _out="${_out%:}"
1135 _err_out="${_out#*monitor:${script_name}:*:*:*:*:}"
1136 case "$_status" in
1137 OK) : ;; # Do nothing special.
1138 TIMEDOUT)
1139 # Recast this as an error, since we can't exit with the
1140 # correct negative number.
1141 _code=1
1142 _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1143 ;;
1144 DISABLED)
1145 # Recast this as an OK, since we can't exit with the
1146 # correct negative number.
1147 _code=0
1148 _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1149 ;;
1150 *) : ;; # Must be ERROR, do nothing special.
1151 esac
1152 if [ -n "$_err_out" ] ; then
1153 echo "$_err_out"
1154 fi
1155 exit $_code
1156 }
1158 ctdb_service_check_reconfigure ()
1159 {
1160 assert_service_name
1162 # We only care about some events in this function. For others we
1163 # return now.
1164 case "$event_name" in
1165 monitor|ipreallocated|reconfigure) : ;;
1166 *) return 0 ;;
1167 esac
1169 if ctdb_reconfigure_take_lock ; then
1170 # No events covered by this function are running, so proceed
1171 # with gay abandon.
1172 case "$event_name" in
1173 reconfigure)
1174 (ctdb_service_reconfigure)
1175 exit $?
1176 ;;
1177 ipreallocated)
1178 if ctdb_service_needs_reconfigure ; then
1179 ctdb_service_reconfigure
1180 fi
1181 ;;
1182 esac
1184 ctdb_reconfigure_release_lock
1185 else
1186 # Somebody else is running an event we don't want to collide
1187 # with. We proceed with caution.
1188 case "$event_name" in
1189 reconfigure)
1190 # Tell whoever called us to retry.
1191 exit 2
1192 ;;
1193 ipreallocated)
1194 # Defer any scheduled reconfigure and just run the
1195 # rest of the ipreallocated event, as per the
1196 # eventscript. There's an assumption here that the
1197 # event doesn't depend on any scheduled reconfigure.
1198 # This is true in the current code.
1199 return 0
1200 ;;
1201 monitor)
1202 # There is most likely a reconfigure in progress so
1203 # the service is possibly unstable. As above, we
1204 # defer any scheduled reconfigured. We also replay
1205 # the previous monitor status since that's the best
1206 # information we have.
1207 ctdb_replay_monitor_status
1208 ;;
1209 esac
1210 fi
1211 }
1213 ##################################################################
1214 # Does CTDB manage this service? - and associated auto-start/stop
1216 ctdb_compat_managed_service ()
1217 {
1218 if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1219 CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1220 fi
1221 }
1223 is_ctdb_managed_service ()
1224 {
1225 assert_service_name
1227 # $t is used just for readability and to allow better accurate
1228 # matching via leading/trailing spaces
1229 t=" $CTDB_MANAGED_SERVICES "
1231 # Return 0 if "<space>$service_name<space>" appears in $t
1232 if [ "${t#* ${service_name} }" != "${t}" ] ; then
1233 return 0
1234 fi
1236 # If above didn't match then update $CTDB_MANAGED_SERVICES for
1237 # backward compatibility and try again.
1238 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
1239 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
1240 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
1241 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "apache2"
1242 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
1243 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
1244 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
1245 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
1246 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
1248 t=" $CTDB_MANAGED_SERVICES "
1250 # Return 0 if "<space>$service_name<space>" appears in $t
1251 [ "${t#* ${service_name} }" != "${t}" ]
1252 }
1254 ctdb_start_stop_service ()
1255 {
1256 assert_service_name
1258 # Allow service-start/service-stop pseudo-events to start/stop
1259 # services when we're not auto-starting/stopping and we're not
1260 # monitoring.
1261 case "$event_name" in
1262 service-start)
1263 if is_ctdb_managed_service ; then
1264 die 'service-start event not permitted when service is managed'
1265 fi
1266 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1267 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1268 fi
1269 ctdb_service_start
1270 exit $?
1271 ;;
1272 service-stop)
1273 if is_ctdb_managed_service ; then
1274 die 'service-stop event not permitted when service is managed'
1275 fi
1276 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1277 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1278 fi
1279 ctdb_service_stop
1280 exit $?
1281 ;;
1282 esac
1284 # Do nothing unless configured to...
1285 [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1287 [ "$event_name" = "monitor" ] || return 0
1289 if is_ctdb_managed_service ; then
1290 if ! is_ctdb_previously_managed_service ; then
1291 echo "Starting service \"$service_name\" - now managed"
1292 background_with_logging ctdb_service_start
1293 exit $?
1294 fi
1295 else
1296 if is_ctdb_previously_managed_service ; then
1297 echo "Stopping service \"$service_name\" - no longer managed"
1298 background_with_logging ctdb_service_stop
1299 exit $?
1300 fi
1301 fi
1302 }
1304 ctdb_service_start ()
1305 {
1306 # The service is marked managed if we've ever tried to start it.
1307 ctdb_service_managed
1309 service_start || return $?
1311 ctdb_counter_init
1312 ctdb_check_tcp_init
1313 }
1315 ctdb_service_stop ()
1316 {
1317 ctdb_service_unmanaged
1318 service_stop
1319 }
1321 # Default service_start() and service_stop() functions.
1323 # These may be overridden in an eventscript.
1324 service_start ()
1325 {
1326 service "$service_name" start
1327 }
1329 service_stop ()
1330 {
1331 service "$service_name" stop
1332 }
1334 ##################################################################
1336 ctdb_standard_event_handler ()
1337 {
1338 case "$1" in
1339 status)
1340 ctdb_checkstatus
1341 exit
1342 ;;
1343 setstatus)
1344 shift
1345 ctdb_setstatus "$@"
1346 exit
1347 ;;
1348 esac
1349 }
1351 # iptables doesn't like being re-entered, so flock-wrap it.
1352 iptables()
1353 {
1354 flock -w 30 $CTDB_VARDIR/iptables-ctdb.flock /sbin/iptables "$@"
1355 }
1357 # AIX (and perhaps others?) doesn't have mktemp
1358 if ! which mktemp >/dev/null 2>&1 ; then
1359 mktemp ()
1360 {
1361 _dir=false
1362 if [ "$1" = "-d" ] ; then
1363 _dir=true
1364 shift
1365 fi
1366 _d="${TMPDIR:-/tmp}"
1367 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1368 md5sum | \
1369 sed -e 's@\(..........\).*@\1@')
1370 _t="${_d}/tmp.${_hex10}"
1371 (
1372 umask 077
1373 if $_dir ; then
1374 mkdir "$_t"
1375 else
1376 >"$_t"
1377 fi
1378 )
1379 echo "$_t"
1380 }
1381 fi
1383 ########################################################
1384 # tickle handling
1385 ########################################################
1387 update_tickles ()
1388 {
1389 _port="$1"
1391 tickledir="$CTDB_VARDIR/state/tickles"
1392 mkdir -p "$tickledir"
1394 # Who am I?
1395 _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1397 # What public IPs do I hold?
1398 _ips=$(ctdb -Y ip | awk -F: -v pnn=$_pnn '$3 == pnn {print $2}')
1400 # IPs as a regexp choice
1401 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1403 # Record connections to our public IPs in a temporary file
1404 _my_connections="${tickledir}/${_port}.connections"
1405 rm -f "$_my_connections"
1406 netstat -tn |
1407 awk -v destpat="^${_ipschoice}:${_port}\$" \
1408 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1409 sort >"$_my_connections"
1411 # Record our current tickles in a temporary file
1412 _my_tickles="${tickledir}/${_port}.tickles"
1413 rm -f "$_my_tickles"
1414 for _i in $_ips ; do
1415 ctdb -Y gettickles $_i $_port |
1416 awk -F: 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1417 done |
1418 sort >"$_my_tickles"
1420 # Add tickles for connections that we haven't already got tickles for
1421 comm -23 "$_my_connections" "$_my_tickles" |
1422 while read _src _dst ; do
1423 ctdb addtickle $_src $_dst
1424 done
1426 # Remove tickles for connections that are no longer there
1427 comm -13 "$_my_connections" "$_my_tickles" |
1428 while read _src _dst ; do
1429 ctdb deltickle $_src $_dst
1430 done
1432 rm -f "$_my_connections" "$_my_tickles"
1433 }
1435 ########################################################
1436 # load a site local config file
1437 ########################################################
1439 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1440 . "$CTDB_RC_LOCAL"
1441 }
1443 [ -x $CTDB_BASE/rc.local ] && {
1444 . $CTDB_BASE/rc.local
1445 }
1447 [ -d $CTDB_BASE/rc.local.d ] && {
1448 for i in $CTDB_BASE/rc.local.d/* ; do
1449 [ -x "$i" ] && . "$i"
1450 done
1451 }
1453 script_name="${0##*/}" # basename
1454 service_fail_limit=1
1455 event_name="$1"