| blob | ec695830713d610413b0a47bdb7f37110e9ef418 |
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6 if [ -d "/var/lib/ctdb" ] ; then
7 export CTDB_VARDIR="/var/lib/ctdb"
8 else
9 export CTDB_VARDIR="/var/ctdb"
10 fi
11 }
12 [ -z "$CTDB_ETCDIR" ] && {
13 export CTDB_ETCDIR="/etc"
14 }
16 #######################################
17 # pull in a system config file, if any
18 _loadconfig() {
20 if [ -z "$1" ] ; then
21 foo="${service_config:-${service_name}}"
22 if [ -n "$foo" ] ; then
23 loadconfig "$foo"
24 return
25 fi
26 fi
28 if [ "$1" != "ctdb" ] ; then
29 loadconfig "ctdb"
30 fi
32 if [ -z "$1" ] ; then
33 return
34 fi
36 if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37 . $CTDB_ETCDIR/sysconfig/$1
38 elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39 . $CTDB_ETCDIR/default/$1
40 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41 . $CTDB_BASE/sysconfig/$1
42 fi
44 if [ "$1" = "ctdb" ] ; then
45 _config="${CTDB_BASE}/ctdbd.conf"
46 if [ -r "$_config" ] ; then
47 . "$_config"
48 fi
49 fi
50 }
52 loadconfig () {
53 _loadconfig "$@"
54 }
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
59 # configuration file.
60 debug ()
61 {
62 if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63 # If there are arguments then echo them. Otherwise expect to
64 # use stdin, which allows us to pass lots of debug using a
65 # here document.
66 if [ -n "$1" ] ; then
67 echo "DEBUG: $*"
68 else
69 sed -e 's@^@DEBUG: @'
70 fi
71 else
72 if [ -z "$1" ] ; then
73 cat >/dev/null
74 fi
75 fi
76 }
78 die ()
79 {
80 _msg="$1"
81 _rc="${2:-1}"
83 echo "$_msg"
84 exit $_rc
85 }
87 # Log given message or stdin to either syslog or a CTDB log file
88 # $1 is the tag passed to logger if syslog is in use.
89 script_log ()
90 {
91 _tag="$1" ; shift
93 case "$CTDB_LOGGING" in
94 file:*|"")
95 if [ -n "$CTDB_LOGGING" ] ; then
96 _file="${CTDB_LOGGING#file:}"
97 else
98 _file="/var/log/log.ctdb"
99 fi
100 {
101 if [ -n "$*" ] ; then
102 echo "$*"
103 else
104 cat
105 fi
106 } >>"$_file"
107 ;;
108 *)
109 # Handle all syslog:* variants here too. There's no tool to do
110 # the lossy things, so just use logger.
111 logger -t "ctdbd: ${_tag}" $*
112 ;;
113 esac
114 }
116 # When things are run in the background in an eventscript then logging
117 # output might get lost. This is the "solution". :-)
118 background_with_logging ()
119 {
120 (
121 "$@" 2>&1 </dev/null |
122 script_log "${script_name}&"
123 )&
125 return 0
126 }
128 ##############################################################
129 # check number of args for different events
130 ctdb_check_args ()
131 {
132 case "$1" in
133 takeip|releaseip)
134 if [ $# != 4 ]; then
135 echo "ERROR: must supply interface, IP and maskbits"
136 exit 1
137 fi
138 ;;
139 updateip)
140 if [ $# != 5 ]; then
141 echo "ERROR: must supply old interface, new interface, IP and maskbits"
142 exit 1
143 fi
144 ;;
145 esac
146 }
148 ##############################################################
149 # determine on what type of system (init style) we are running
150 detect_init_style()
151 {
152 # only do detection if not already set:
153 [ -z "$CTDB_INIT_STYLE" ] || return
155 if [ -x /sbin/startproc ]; then
156 CTDB_INIT_STYLE="suse"
157 elif [ -x /sbin/start-stop-daemon ]; then
158 CTDB_INIT_STYLE="debian"
159 else
160 CTDB_INIT_STYLE="redhat"
161 fi
162 }
164 ######################################################
165 # simulate /sbin/service on platforms that don't have it
166 # _service() makes it easier to hook the service() function for
167 # testing.
168 _service ()
169 {
170 _service_name="$1"
171 _op="$2"
173 # do nothing, when no service was specified
174 [ -z "$_service_name" ] && return
176 if [ -x /sbin/service ]; then
177 $_nice /sbin/service "$_service_name" "$_op"
178 elif [ -x /usr/sbin/service ]; then
179 $_nice /usr/sbin/service "$_service_name" "$_op"
180 elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
181 $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
182 elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
183 $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
184 fi
185 }
187 service()
188 {
189 _nice=""
190 _service "$@"
191 }
193 ######################################################
194 # simulate /sbin/service (niced) on platforms that don't have it
195 nice_service()
196 {
197 _nice="nice"
198 _service "$@"
199 }
201 ######################################################
202 # Cached retrieval of PNN from local node. This never changes so why
203 # open a client connection to the server each time this is needed?
204 # This sets $pnn - this avoid an unnecessary subprocess.
205 ctdb_get_pnn ()
206 {
207 _pnn_file="$CTDB_VARDIR/state/my-pnn"
208 if [ ! -f "$_pnn_file" ] ; then
209 ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
210 fi
212 read pnn <"$_pnn_file"
213 }
215 ######################################################
216 # wrapper around /proc/ settings to allow them to be hooked
217 # for testing
218 # 1st arg is relative path under /proc/, 2nd arg is value to set
219 set_proc ()
220 {
221 echo "$2" >"/proc/$1"
222 }
224 ######################################################
225 # wrapper around getting file contents from /proc/ to allow
226 # this to be hooked for testing
227 # 1st arg is relative path under /proc/
228 get_proc ()
229 {
230 cat "/proc/$1"
231 }
233 ######################################################
234 # Print up to $_max kernel stack traces for processes named $_program
235 program_stack_traces ()
236 {
237 _prog="$1"
238 _max="${2:-1}"
240 _count=1
241 for _pid in $(pidof "$_prog") ; do
242 [ $_count -le $_max ] || break
244 # Do this first to avoid racing with process exit
245 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
246 if [ -n "$_stack" ] ; then
247 echo "Stack trace for ${_prog}[${_pid}]:"
248 echo "$_stack"
249 _count=$(($_count + 1))
250 fi
251 done
252 }
254 ######################################################
255 # Check that an RPC service is healthy -
256 # this includes allowing a certain number of failures
257 # before marking the NFS service unhealthy.
258 #
259 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
260 #
261 # each triple is a set of 3 arguments: an operator, a
262 # fail count limit and an action string.
263 #
264 # For example:
265 #
266 # nfs_check_rpc_service "lockd" \
267 # -ge 15 "verbose restart unhealthy" \
268 # -eq 10 "restart:bs"
269 #
270 # says that if lockd is down for 15 iterations then do
271 # a verbose restart of lockd and mark the node unhealthy.
272 # Before this, after 10 iterations of failure, the
273 # service is restarted silently in the background.
274 # Order is important: the number of failures need to be
275 # specified in reverse order because processing stops
276 # after the first condition that is true.
277 ######################################################
278 nfs_check_rpc_service ()
279 {
280 _prog_name="$1" ; shift
282 if _nfs_check_rpc_common "$_prog_name" ; then
283 return
284 fi
286 while [ -n "$3" ] ; do
287 if _nfs_check_rpc_action "$1" "$2" "$3" ; then
288 break
289 fi
290 shift 3
291 done
292 }
294 # The new way of doing things...
295 nfs_check_rpc_services ()
296 {
297 # Files must end with .check - avoids editor backups, RPM fu, ...
298 for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
299 _t="${_f%.check}"
300 _prog_name="${_t##*/[0-9][0-9].}"
302 # If $_prog_name contains '@' then the bit after it is the
303 # address family.
304 _family="${_prog_name#*@}"
305 if [ "$_family" = "$_prog_name" ] ; then
306 _family=""
307 else
308 _prog_name="${_prog_name%@*}"
309 fi
311 if _nfs_check_rpc_common "$_prog_name" "$_family" ; then
312 # This RPC service is up, check next service...
313 continue
314 fi
316 # Check each line in the file in turn until one of the limit
317 # checks is hit...
318 while read _cmp _lim _rest ; do
319 # Skip comments
320 case "$_cmp" in
321 \#*) continue ;;
322 esac
324 if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
325 # Limit was hit on this line, no further checking...
326 break
327 fi
328 done <"$_f"
329 done
330 }
332 _nfs_check_rpc_common ()
333 {
334 _prog_name="$1"
335 _family="$2"
337 # Some platforms don't have separate programs for all services.
338 case "$_prog_name" in
339 statd)
340 type "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
341 esac
343 case "$_prog_name" in
344 nfsd)
345 _rpc_prog=nfs
346 _version=3
347 ;;
348 mountd)
349 _rpc_prog=mountd
350 _version=1
351 ;;
352 rquotad)
353 _rpc_prog=rquotad
354 _version=1
355 ;;
356 lockd)
357 _rpc_prog=nlockmgr
358 _version=4
359 ;;
360 statd)
361 _rpc_prog=status
362 _version=1
363 ;;
364 *)
365 echo "Internal error: unknown RPC program \"$_prog_name\"."
366 exit 1
367 esac
369 _service_name="nfs_${_prog_name}${_family:+_}${_family}"
371 if ctdb_check_rpc "$_rpc_prog" "$_version" "$_family" >/dev/null ; then
372 ctdb_counter_init "$_service_name"
373 return 0
374 fi
376 ctdb_counter_incr "$_service_name"
378 return 1
379 }
381 _nfs_check_rpc_action ()
382 {
383 _cmp="$1"
384 _limit="$2"
385 _actions="$3"
387 if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
388 return 1
389 fi
391 for _action in $_actions ; do
392 case "$_action" in
393 verbose)
394 echo "$ctdb_check_rpc_out"
395 ;;
396 restart)
397 _nfs_restart_rpc_service "$_prog_name"
398 ;;
399 restart:b)
400 _nfs_restart_rpc_service "$_prog_name" true
401 ;;
402 unhealthy)
403 exit 1
404 ;;
405 *)
406 echo "Internal error: unknown action \"$_action\"."
407 exit 1
408 esac
409 done
411 return 0
412 }
414 _nfs_restart_rpc_service ()
415 {
416 _prog_name="$1"
417 _background="${2:-false}"
419 if $_background ; then
420 _maybe_background="background_with_logging"
421 else
422 _maybe_background=""
423 fi
425 _p="rpc.${_prog_name}"
427 case "$_prog_name" in
428 nfsd)
429 echo "Trying to restart NFS service"
430 $_maybe_background startstop_nfs restart
431 ;;
432 mountd)
433 echo "Trying to restart $_prog_name [${_p}]"
434 killall -q -9 "$_p"
435 nfs_dump_some_threads "$_p"
436 $_maybe_background $_p $RPCMOUNTDOPTS \
437 ${MOUNTD_PORT:+-p} $MOUNTD_PORT
438 ;;
439 rquotad)
440 echo "Trying to restart $_prog_name [${_p}]"
441 killall -q -9 "$_p"
442 nfs_dump_some_threads "$_p"
443 $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
444 ;;
445 lockd)
446 echo "Trying to restart lock manager service"
447 $_maybe_background startstop_nfslock restart
448 ;;
449 statd)
450 echo "Trying to restart $_prog_name [${_p}]"
451 killall -q -9 "$_p"
452 nfs_dump_some_threads "$_p"
453 $_maybe_background $_p \
454 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
455 ${STATD_PORT:+-p} $STATD_PORT \
456 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
457 ;;
458 *)
459 echo "Internal error: unknown RPC program \"$_prog_name\"."
460 exit 1
461 esac
462 }
464 ######################################################
465 # check that a rpc server is registered with portmap
466 # and responding to requests
467 # usage: ctdb_check_rpc SERVICE_NAME VERSION
468 ######################################################
469 ctdb_check_rpc ()
470 {
471 progname="$1"
472 version="$2"
473 _family="${3:-tcp}"
475 _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
477 if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost $progname $version 2>&1) ; then
478 ctdb_check_rpc_out="ERROR: $progname failed RPC check:
479 $ctdb_check_rpc_out"
480 echo "$ctdb_check_rpc_out"
481 return 1
482 fi
483 }
485 ######################################################
486 # Ensure $service_name is set
487 assert_service_name ()
488 {
489 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
490 }
492 ######################################################
493 # check a set of directories is available
494 # return 1 on a missing directory
495 # directories are read from stdin
496 ######################################################
497 ctdb_check_directories_probe()
498 {
499 while IFS="" read d ; do
500 case "$d" in
501 *%*)
502 continue
503 ;;
504 *)
505 [ -d "${d}/." ] || return 1
506 esac
507 done
508 }
510 ######################################################
511 # check a set of directories is available
512 # directories are read from stdin
513 ######################################################
514 ctdb_check_directories()
515 {
516 ctdb_check_directories_probe || {
517 echo "ERROR: $service_name directory \"$d\" not available"
518 exit 1
519 }
520 }
522 ######################################################
523 # check a set of tcp ports
524 # usage: ctdb_check_tcp_ports <ports...>
525 ######################################################
527 # This flag file is created when a service is initially started. It
528 # is deleted the first time TCP port checks for that service succeed.
529 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
530 # message if a port check fails.
531 _ctdb_check_tcp_common ()
532 {
533 assert_service_name
534 _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
535 }
537 ctdb_check_tcp_init ()
538 {
539 _ctdb_check_tcp_common
540 mkdir -p "${_ctdb_service_started_file%/*}" # dirname
541 touch "$_ctdb_service_started_file"
542 }
544 # Check whether something is listening on all of the given TCP ports
545 # using the "ctdb checktcpport" command.
546 ctdb_check_tcp_ports()
547 {
548 if [ -z "$1" ] ; then
549 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
550 exit 1
551 fi
553 for _p ; do # process each function argument (port)
554 _cmd="ctdb checktcpport $_p"
555 _out=$($_cmd 2>&1)
556 _ret=$?
557 case "$_ret" in
558 0)
559 _ctdb_check_tcp_common
560 if [ ! -f "$_ctdb_service_started_file" ] ; then
561 echo "ERROR: $service_name tcp port $_p is not responding"
562 debug "\"ctdb checktcpport $_p\" was able to bind to port"
563 else
564 echo "INFO: $service_name tcp port $_p is not responding"
565 fi
567 return 1
568 ;;
569 98)
570 # Couldn't bind, something already listening, next port...
571 continue
572 ;;
573 *)
574 echo "ERROR: unexpected error running \"ctdb checktcpport\""
575 debug <<EOF
576 ctdb checktcpport (exited with $_ret) with output:
577 $_out"
578 EOF
579 return $_ret
580 esac
581 done
583 # All ports listening
584 _ctdb_check_tcp_common
585 rm -f "$_ctdb_service_started_file"
586 return 0
587 }
589 ######################################################
590 # check a unix socket
591 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
592 ######################################################
593 ctdb_check_unix_socket() {
594 socket_path="$1"
595 [ -z "$socket_path" ] && return
597 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
598 echo "ERROR: $service_name socket $socket_path not found"
599 return 1
600 fi
601 }
603 ######################################################
604 # check a command returns zero status
605 # usage: ctdb_check_command <command>
606 ######################################################
607 ctdb_check_command ()
608 {
609 _out=$("$@" 2>&1) || {
610 echo "ERROR: $* returned error"
611 echo "$_out" | debug
612 exit 1
613 }
614 }
616 ################################################
617 # kill off any TCP connections with the given IP
618 ################################################
619 kill_tcp_connections ()
620 {
621 _ip="$1"
623 _oneway=false
624 if [ "$2" = "oneway" ] ; then
625 _oneway=true
626 fi
628 get_tcp_connections_for_ip "$_ip" | {
629 _killcount=0
630 _connections=""
631 _nl="
632 "
633 while read _dst _src; do
634 _destport="${_dst##*:}"
635 __oneway=$_oneway
636 case $_destport in
637 # we only do one-way killtcp for CIFS
638 139|445) __oneway=true ;;
639 esac
641 echo "Killing TCP connection $_src $_dst"
642 _connections="${_connections}${_nl}${_src} ${_dst}"
643 if ! $__oneway ; then
644 _connections="${_connections}${_nl}${_dst} ${_src}"
645 fi
647 _killcount=$(($_killcount + 1))
648 done
650 if [ $_killcount -eq 0 ] ; then
651 return
652 fi
654 echo "$_connections" | ctdb killtcp || {
655 echo "Failed to send killtcp control"
656 return
657 }
659 _count=0
660 while : ; do
661 _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
663 if [ $_remaining -eq 0 ] ; then
664 echo "Killed $_killcount TCP connections to released IP $_ip"
665 return
666 fi
668 _count=$(($_count + 1))
669 if [ $_count -gt 3 ] ; then
670 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
671 return
672 fi
674 echo "Waiting for $_remaining connections to be killed for IP $_ip"
675 sleep 1
676 done
677 }
678 }
680 ##################################################################
681 # kill off the local end for any TCP connections with the given IP
682 ##################################################################
683 kill_tcp_connections_local_only ()
684 {
685 kill_tcp_connections "$1" "oneway"
686 }
688 ##################################################################
689 # tickle any TCP connections with the given IP
690 ##################################################################
691 tickle_tcp_connections ()
692 {
693 _ip="$1"
695 get_tcp_connections_for_ip "$_ip" |
696 {
697 _failed=false
699 while read dest src; do
700 echo "Tickle TCP connection $src $dest"
701 ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
702 echo "Tickle TCP connection $dest $src"
703 ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
704 done
706 if $_failed ; then
707 echo "Failed to send tickle control"
708 fi
709 }
710 }
712 get_tcp_connections_for_ip ()
713 {
714 _ip="$1"
716 netstat -tn | awk -v ip=$_ip \
717 'index($1, "tcp") == 1 && \
718 (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
719 && $6 == "ESTABLISHED" \
720 {print $4" "$5}'
721 }
723 ##################################################################
724 # use statd-callout to update NFS lock info
725 ##################################################################
726 nfs_update_lock_info ()
727 {
728 if [ -x "$CTDB_BASE/statd-callout" ] ; then
729 "$CTDB_BASE/statd-callout" update
730 fi
731 }
733 ########################################################
734 # start/stop the Ganesha nfs service
735 ########################################################
736 startstop_ganesha()
737 {
738 _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
739 case "$1" in
740 start)
741 service "$_service_name" start
742 ;;
743 stop)
744 service "$_service_name" stop
745 ;;
746 restart)
747 service "$_service_name" stop
748 nfs_dump_some_threads "rpc.statd"
749 service "$_service_name" start
750 ;;
751 esac
752 }
754 ########################################################
755 # start/stop the nfs service on different platforms
756 ########################################################
757 startstop_nfs() {
758 PLATFORM="unknown"
759 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
760 PLATFORM="sles"
761 }
762 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
763 -r /usr/lib/systemd/system/nfs-lock.service ] && {
764 PLATFORM="rhel"
765 }
767 case $PLATFORM in
768 sles)
769 case $1 in
770 start)
771 service nfsserver start
772 ;;
773 stop)
774 service nfsserver stop > /dev/null 2>&1
775 ;;
776 restart)
777 set_proc "fs/nfsd/threads" 0
778 service nfsserver stop > /dev/null 2>&1
779 pkill -9 nfsd
780 nfs_dump_some_threads
781 service nfsserver start
782 ;;
783 esac
784 ;;
785 rhel)
786 case $1 in
787 start)
788 service nfslock start
789 service nfs start
790 ;;
791 stop)
792 service nfs stop
793 service nfslock stop
794 ;;
795 restart)
796 set_proc "fs/nfsd/threads" 0
797 service nfs stop > /dev/null 2>&1
798 service nfslock stop > /dev/null 2>&1
799 pkill -9 nfsd
800 nfs_dump_some_threads
801 service nfslock start
802 service nfs start
803 ;;
804 esac
805 ;;
806 *)
807 echo "Unknown platform. NFS is not supported with ctdb"
808 exit 1
809 ;;
810 esac
811 }
813 # Dump up to the configured number of nfsd thread backtraces.
814 nfs_dump_some_threads ()
815 {
816 _prog="${1:-nfsd}"
818 _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
819 [ $_num -gt 0 ] || return 0
821 program_stack_traces "$_prog" $_num
822 }
824 ########################################################
825 # start/stop the nfs lockmanager service on different platforms
826 ########################################################
827 startstop_nfslock() {
828 PLATFORM="unknown"
829 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
830 PLATFORM="sles"
831 }
832 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
833 -r /usr/lib/systemd/system/nfs-lock.service ] && {
834 PLATFORM="rhel"
835 }
837 case $PLATFORM in
838 sles)
839 # for sles there is no service for lockmanager
840 # so we instead just shutdown/restart nfs
841 case $1 in
842 start)
843 service nfsserver start
844 ;;
845 stop)
846 service nfsserver stop > /dev/null 2>&1
847 ;;
848 restart)
849 service nfsserver stop > /dev/null 2>&1
850 service nfsserver start
851 ;;
852 esac
853 ;;
854 rhel)
855 case $1 in
856 start)
857 service nfslock start
858 ;;
859 stop)
860 service nfslock stop > /dev/null 2>&1
861 ;;
862 restart)
863 service nfslock stop > /dev/null 2>&1
864 service nfslock start
865 ;;
866 esac
867 ;;
868 *)
869 echo "Unknown platform. NFS locking is not supported with ctdb"
870 exit 1
871 ;;
872 esac
873 }
875 ########################################################
877 add_ip_to_iface ()
878 {
879 _iface=$1
880 _ip=$2
881 _maskbits=$3
883 # Ensure interface is up
884 ip link set "$_iface" up || \
885 die "Failed to bringup interface $_iface"
887 # Only need to define broadcast for IPv4
888 case "$ip" in
889 *:*) _bcast="" ;;
890 *) _bcast="brd +" ;;
891 esac
893 ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
894 echo "Failed to add $_ip/$_maskbits on dev $_iface"
895 return 1
896 }
898 # Wait 5 seconds for IPv6 addresses to stop being tentative...
899 if [ -z "$_bcast" ] ; then
900 for _x in $(seq 1 10) ; do
901 ip addr show to "${_ip}/128" | grep -q "tentative" || break
902 sleep 0.5
903 done
905 # If the address was a duplicate then it won't be on the
906 # interface so flag an error.
907 _t=$(ip addr show to "${_ip}/128")
908 case "$_t" in
909 "")
910 echo "Failed to add $_ip/$_maskbits on dev $_iface"
911 return 1
912 ;;
913 *tentative*|*dadfailed*)
914 echo "Failed to add $_ip/$_maskbits on dev $_iface"
915 ip addr del "$_ip/$_maskbits" dev "$_iface"
916 return 1
917 ;;
918 esac
919 fi
920 }
922 delete_ip_from_iface()
923 {
924 _iface=$1
925 _ip=$2
926 _maskbits=$3
928 # This could be set globally for all interfaces but it is probably
929 # better to avoid surprises, so limit it the interfaces where CTDB
930 # has public IP addresses. There isn't anywhere else convenient
931 # to do this so just set it each time. This is much cheaper than
932 # remembering and re-adding secondaries.
933 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
935 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
936 echo "Failed to del $_ip on dev $_iface"
937 return 1
938 }
939 }
941 # If the given IP is hosted then print 2 items: maskbits and iface
942 ip_maskbits_iface ()
943 {
944 _addr="$1"
946 case "$_addr" in
947 *:*) _family="inet6" ; _bits=128 ;;
948 *) _family="inet" ; _bits=32 ;;
949 esac
951 ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
952 awk -v family="${_family}" \
953 'NR == 1 { iface = $2; sub(":$", "", iface) } \
954 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
955 print mask, iface, family }'
956 }
958 drop_ip ()
959 {
960 _addr="${1%/*}" # Remove optional maskbits
962 set -- $(ip_maskbits_iface $_addr)
963 if [ -n "$1" ] ; then
964 _maskbits="$1"
965 _iface="$2"
966 echo "Removing public address $_addr/$_maskbits from device $_iface"
967 delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
968 fi
969 }
971 drop_all_public_ips ()
972 {
973 while read _ip _x ; do
974 drop_ip "$_ip"
975 done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
976 }
978 flush_route_cache ()
979 {
980 set_proc sys/net/ipv4/route/flush 1
981 set_proc sys/net/ipv6/route/flush 1
982 }
984 ########################################################
985 # Simple counters
986 _ctdb_counter_common () {
987 _service_name="${1:-${service_name:-${script_name}}}"
988 _counter_file="$ctdb_fail_dir/$_service_name"
989 mkdir -p "${_counter_file%/*}" # dirname
990 }
991 ctdb_counter_init () {
992 _ctdb_counter_common "$1"
994 >"$_counter_file"
995 }
996 ctdb_counter_incr () {
997 _ctdb_counter_common "$1"
999 # unary counting!
1000 echo -n 1 >> "$_counter_file"
1001 }
1002 ctdb_check_counter () {
1003 _msg="${1:-error}" # "error" - anything else is silent on fail
1004 _op="${2:--ge}" # an integer operator supported by test
1005 _limit="${3:-${service_fail_limit}}"
1006 shift 3
1007 _ctdb_counter_common "$1"
1009 # unary counting!
1010 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
1011 _hit=false
1012 if [ "$_op" != "%" ] ; then
1013 if [ $_size $_op $_limit ] ; then
1014 _hit=true
1015 fi
1016 else
1017 if [ $(($_size $_op $_limit)) -eq 0 ] ; then
1018 _hit=true
1019 fi
1020 fi
1021 if $_hit ; then
1022 if [ "$_msg" = "error" ] ; then
1023 echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1024 exit 1
1025 else
1026 return 1
1027 fi
1028 fi
1029 }
1031 ########################################################
1033 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1034 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1036 ctdb_setup_service_state_dir ()
1037 {
1038 service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1039 mkdir -p "$service_state_dir" || {
1040 echo "Error creating state dir \"$service_state_dir\""
1041 exit 1
1042 }
1043 }
1045 ########################################################
1046 # Managed status history, for auto-start/stop
1048 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1050 _ctdb_managed_common ()
1051 {
1052 _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1053 }
1055 ctdb_service_managed ()
1056 {
1057 _ctdb_managed_common
1058 mkdir -p "$ctdb_managed_dir"
1059 touch "$_ctdb_managed_file"
1060 }
1062 ctdb_service_unmanaged ()
1063 {
1064 _ctdb_managed_common
1065 rm -f "$_ctdb_managed_file"
1066 }
1068 is_ctdb_previously_managed_service ()
1069 {
1070 _ctdb_managed_common
1071 [ -f "$_ctdb_managed_file" ]
1072 }
1074 ########################################################
1075 # Check and set status
1077 log_status_cat ()
1078 {
1079 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1080 }
1082 ctdb_checkstatus ()
1083 {
1084 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1085 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1086 return 1
1087 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1088 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1089 return 2
1090 else
1091 return 0
1092 fi
1093 }
1095 ctdb_setstatus ()
1096 {
1097 d="$ctdb_status_dir/$script_name"
1098 case "$1" in
1099 unhealthy|banned)
1100 mkdir -p "$d"
1101 cat "$2" >"$d/$1"
1102 ;;
1103 *)
1104 for i in "banned" "unhealthy" ; do
1105 rm -f "$d/$i"
1106 done
1107 ;;
1108 esac
1109 }
1111 ##################################################################
1112 # Reconfigure a service on demand
1114 _ctdb_service_reconfigure_common ()
1115 {
1116 _d="$ctdb_status_dir/${service_name}"
1117 mkdir -p "$_d"
1118 _ctdb_service_reconfigure_flag="$_d/reconfigure"
1119 }
1121 ctdb_service_needs_reconfigure ()
1122 {
1123 _ctdb_service_reconfigure_common
1124 [ -e "$_ctdb_service_reconfigure_flag" ]
1125 }
1127 ctdb_service_set_reconfigure ()
1128 {
1129 _ctdb_service_reconfigure_common
1130 >"$_ctdb_service_reconfigure_flag"
1131 }
1133 ctdb_service_unset_reconfigure ()
1134 {
1135 _ctdb_service_reconfigure_common
1136 rm -f "$_ctdb_service_reconfigure_flag"
1137 }
1139 ctdb_service_reconfigure ()
1140 {
1141 echo "Reconfiguring service \"${service_name}\"..."
1142 ctdb_service_unset_reconfigure
1143 service_reconfigure || return $?
1144 ctdb_counter_init
1145 }
1147 # Default service_reconfigure() function does nothing.
1148 service_reconfigure ()
1149 {
1150 :
1151 }
1153 ctdb_reconfigure_take_lock ()
1154 {
1155 _ctdb_service_reconfigure_common
1156 _lock="${_d}/reconfigure_lock"
1157 mkdir -p "${_lock%/*}" # dirname
1158 touch "$_lock"
1160 (
1161 flock 0
1162 # This is overkill but will work if we need to extend this to
1163 # allow certain events to run multiple times in parallel
1164 # (e.g. takeip) and write multiple PIDs to the file.
1165 read _locker_event
1166 if [ -n "$_locker_event" ] ; then
1167 while read _pid ; do
1168 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1169 kill -0 "$_pid" 2>/dev/null ; then
1170 exit 1
1171 fi
1172 done
1173 fi
1175 printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1176 exit 0
1177 ) <"$_lock"
1178 }
1180 ctdb_reconfigure_release_lock ()
1181 {
1182 _ctdb_service_reconfigure_common
1183 _lock="${_d}/reconfigure_lock"
1185 rm -f "$_lock"
1186 }
1188 ctdb_replay_monitor_status ()
1189 {
1190 echo "Replaying previous status for this script due to reconfigure..."
1191 # Leading separator ('|') is missing in some versions...
1192 _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1193 # Output looks like this:
1194 # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1195 # This is the cheapest way of getting fields in the middle.
1196 set -- $(IFS="|" ; echo $_out)
1197 _code="$3"
1198 _status="$4"
1199 # The error output field can include colons so we'll try to
1200 # preserve them. The weak checking at the beginning tries to make
1201 # this work for both broken (no leading '|') and fixed output.
1202 _out="${_out%|}"
1203 _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1204 case "$_status" in
1205 OK) : ;; # Do nothing special.
1206 TIMEDOUT)
1207 # Recast this as an error, since we can't exit with the
1208 # correct negative number.
1209 _code=1
1210 _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1211 ;;
1212 DISABLED)
1213 # Recast this as an OK, since we can't exit with the
1214 # correct negative number.
1215 _code=0
1216 _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1217 ;;
1218 *) : ;; # Must be ERROR, do nothing special.
1219 esac
1220 if [ -n "$_err_out" ] ; then
1221 echo "$_err_out"
1222 fi
1223 exit $_code
1224 }
1226 ctdb_service_check_reconfigure ()
1227 {
1228 assert_service_name
1230 # We only care about some events in this function. For others we
1231 # return now.
1232 case "$event_name" in
1233 monitor|ipreallocated|reconfigure) : ;;
1234 *) return 0 ;;
1235 esac
1237 if ctdb_reconfigure_take_lock ; then
1238 # No events covered by this function are running, so proceed
1239 # with gay abandon.
1240 case "$event_name" in
1241 reconfigure)
1242 (ctdb_service_reconfigure)
1243 exit $?
1244 ;;
1245 ipreallocated)
1246 if ctdb_service_needs_reconfigure ; then
1247 ctdb_service_reconfigure
1248 fi
1249 ;;
1250 esac
1252 ctdb_reconfigure_release_lock
1253 else
1254 # Somebody else is running an event we don't want to collide
1255 # with. We proceed with caution.
1256 case "$event_name" in
1257 reconfigure)
1258 # Tell whoever called us to retry.
1259 exit 2
1260 ;;
1261 ipreallocated)
1262 # Defer any scheduled reconfigure and just run the
1263 # rest of the ipreallocated event, as per the
1264 # eventscript. There's an assumption here that the
1265 # event doesn't depend on any scheduled reconfigure.
1266 # This is true in the current code.
1267 return 0
1268 ;;
1269 monitor)
1270 # There is most likely a reconfigure in progress so
1271 # the service is possibly unstable. As above, we
1272 # defer any scheduled reconfigured. We also replay
1273 # the previous monitor status since that's the best
1274 # information we have.
1275 ctdb_replay_monitor_status
1276 ;;
1277 esac
1278 fi
1279 }
1281 ##################################################################
1282 # Does CTDB manage this service? - and associated auto-start/stop
1284 ctdb_compat_managed_service ()
1285 {
1286 if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1287 CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1288 fi
1289 }
1291 is_ctdb_managed_service ()
1292 {
1293 assert_service_name
1295 # $t is used just for readability and to allow better accurate
1296 # matching via leading/trailing spaces
1297 t=" $CTDB_MANAGED_SERVICES "
1299 # Return 0 if "<space>$service_name<space>" appears in $t
1300 if [ "${t#* ${service_name} }" != "${t}" ] ; then
1301 return 0
1302 fi
1304 # If above didn't match then update $CTDB_MANAGED_SERVICES for
1305 # backward compatibility and try again.
1306 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
1307 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
1308 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
1309 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "apache2"
1310 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
1311 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
1312 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
1313 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
1314 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
1316 t=" $CTDB_MANAGED_SERVICES "
1318 # Return 0 if "<space>$service_name<space>" appears in $t
1319 [ "${t#* ${service_name} }" != "${t}" ]
1320 }
1322 ctdb_start_stop_service ()
1323 {
1324 assert_service_name
1326 # Allow service-start/service-stop pseudo-events to start/stop
1327 # services when we're not auto-starting/stopping and we're not
1328 # monitoring.
1329 case "$event_name" in
1330 service-start)
1331 if is_ctdb_managed_service ; then
1332 die 'service-start event not permitted when service is managed'
1333 fi
1334 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1335 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1336 fi
1337 ctdb_service_start
1338 exit $?
1339 ;;
1340 service-stop)
1341 if is_ctdb_managed_service ; then
1342 die 'service-stop event not permitted when service is managed'
1343 fi
1344 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1345 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1346 fi
1347 ctdb_service_stop
1348 exit $?
1349 ;;
1350 esac
1352 # Do nothing unless configured to...
1353 [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1355 [ "$event_name" = "monitor" ] || return 0
1357 if is_ctdb_managed_service ; then
1358 if ! is_ctdb_previously_managed_service ; then
1359 echo "Starting service \"$service_name\" - now managed"
1360 background_with_logging ctdb_service_start
1361 exit $?
1362 fi
1363 else
1364 if is_ctdb_previously_managed_service ; then
1365 echo "Stopping service \"$service_name\" - no longer managed"
1366 background_with_logging ctdb_service_stop
1367 exit $?
1368 fi
1369 fi
1370 }
1372 ctdb_service_start ()
1373 {
1374 # The service is marked managed if we've ever tried to start it.
1375 ctdb_service_managed
1377 service_start || return $?
1379 ctdb_counter_init
1380 ctdb_check_tcp_init
1381 }
1383 ctdb_service_stop ()
1384 {
1385 ctdb_service_unmanaged
1386 service_stop
1387 }
1389 # Default service_start() and service_stop() functions.
1391 # These may be overridden in an eventscript.
1392 service_start ()
1393 {
1394 service "$service_name" start
1395 }
1397 service_stop ()
1398 {
1399 service "$service_name" stop
1400 }
1402 ##################################################################
1404 ctdb_standard_event_handler ()
1405 {
1406 case "$1" in
1407 status)
1408 ctdb_checkstatus
1409 exit
1410 ;;
1411 setstatus)
1412 shift
1413 ctdb_setstatus "$@"
1414 exit
1415 ;;
1416 esac
1417 }
1419 iptables_wrapper ()
1420 {
1421 _family="$1" ; shift
1422 if [ "$_family" = "inet6" ] ; then
1423 _iptables_cmd="ip6tables"
1424 else
1425 _iptables_cmd="iptables"
1426 fi
1428 # iptables doesn't like being re-entered, so flock-wrap it.
1429 flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1430 }
1432 # AIX (and perhaps others?) doesn't have mktemp
1433 if ! type mktemp >/dev/null 2>&1 ; then
1434 mktemp ()
1435 {
1436 _dir=false
1437 if [ "$1" = "-d" ] ; then
1438 _dir=true
1439 shift
1440 fi
1441 _d="${TMPDIR:-/tmp}"
1442 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1443 md5sum | \
1444 sed -e 's@\(..........\).*@\1@')
1445 _t="${_d}/tmp.${_hex10}"
1446 (
1447 umask 077
1448 if $_dir ; then
1449 mkdir "$_t"
1450 else
1451 >"$_t"
1452 fi
1453 )
1454 echo "$_t"
1455 }
1456 fi
1458 ########################################################
1459 # tickle handling
1460 ########################################################
1462 update_tickles ()
1463 {
1464 _port="$1"
1466 tickledir="$CTDB_VARDIR/state/tickles"
1467 mkdir -p "$tickledir"
1469 ctdb_get_pnn
1471 # What public IPs do I hold?
1472 _ips=$(ctdb -X ip | awk -F'|' -v pnn=$pnn '$3 == pnn {print $2}')
1474 # IPs as a regexp choice
1475 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1477 # Record connections to our public IPs in a temporary file
1478 _my_connections="${tickledir}/${_port}.connections"
1479 rm -f "$_my_connections"
1480 netstat -tn |
1481 awk -v destpat="^${_ipschoice}:${_port}\$" \
1482 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1483 sort >"$_my_connections"
1485 # Record our current tickles in a temporary file
1486 _my_tickles="${tickledir}/${_port}.tickles"
1487 rm -f "$_my_tickles"
1488 for _i in $_ips ; do
1489 ctdb -X gettickles $_i $_port |
1490 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1491 done |
1492 sort >"$_my_tickles"
1494 # Add tickles for connections that we haven't already got tickles for
1495 comm -23 "$_my_connections" "$_my_tickles" |
1496 while read _src _dst ; do
1497 ctdb addtickle $_src $_dst
1498 done
1500 # Remove tickles for connections that are no longer there
1501 comm -13 "$_my_connections" "$_my_tickles" |
1502 while read _src _dst ; do
1503 ctdb deltickle $_src $_dst
1504 done
1506 rm -f "$_my_connections" "$_my_tickles"
1507 }
1509 ########################################################
1510 # load a site local config file
1511 ########################################################
1513 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1514 . "$CTDB_RC_LOCAL"
1515 }
1517 [ -x $CTDB_BASE/rc.local ] && {
1518 . $CTDB_BASE/rc.local
1519 }
1521 [ -d $CTDB_BASE/rc.local.d ] && {
1522 for i in $CTDB_BASE/rc.local.d/* ; do
1523 [ -x "$i" ] && . "$i"
1524 done
1525 }
1527 script_name="${0##*/}" # basename
1528 service_fail_limit=1
1529 event_name="$1"