| blob | 9623514f2fbe6a27b49d66ba212f36d6742e5db6 |
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6 if [ -d "/var/lib/ctdb" ] ; then
7 export CTDB_VARDIR="/var/lib/ctdb"
8 else
9 export CTDB_VARDIR="/var/ctdb"
10 fi
11 }
12 [ -z "$CTDB_ETCDIR" ] && {
13 export CTDB_ETCDIR="/etc"
14 }
16 #######################################
17 # pull in a system config file, if any
18 _loadconfig() {
20 if [ -z "$1" ] ; then
21 foo="${service_config:-${service_name}}"
22 if [ -n "$foo" ] ; then
23 loadconfig "$foo"
24 return
25 fi
26 fi
28 if [ "$1" != "ctdb" ] ; then
29 loadconfig "ctdb"
30 fi
32 if [ -z "$1" ] ; then
33 return
34 fi
36 if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37 . $CTDB_ETCDIR/sysconfig/$1
38 elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39 . $CTDB_ETCDIR/default/$1
40 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41 . $CTDB_BASE/sysconfig/$1
42 fi
44 if [ "$1" = "ctdb" ] ; then
45 _config="${CTDB_BASE}/ctdbd.conf"
46 if [ -r "$_config" ] ; then
47 . "$_config"
48 fi
49 fi
50 }
52 loadconfig () {
53 _loadconfig "$@"
54 }
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
59 # configuration file.
60 debug ()
61 {
62 if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63 # If there are arguments then echo them. Otherwise expect to
64 # use stdin, which allows us to pass lots of debug using a
65 # here document.
66 if [ -n "$1" ] ; then
67 echo "DEBUG: $*"
68 else
69 sed -e 's@^@DEBUG: @'
70 fi
71 else
72 if [ -z "$1" ] ; then
73 cat >/dev/null
74 fi
75 fi
76 }
78 die ()
79 {
80 _msg="$1"
81 _rc="${2:-1}"
83 echo "$_msg"
84 exit $_rc
85 }
87 # Log given message or stdin to either syslog or a CTDB log file
88 # $1 is the tag passed to logger if syslog is in use.
89 script_log ()
90 {
91 _tag="$1" ; shift
93 case "$CTDB_LOGGING" in
94 file:*|"")
95 if [ -n "$CTDB_LOGGING" ] ; then
96 _file="${CTDB_LOGGING#file:}"
97 else
98 _file="/var/log/log.ctdb"
99 fi
100 {
101 if [ -n "$*" ] ; then
102 echo "$*"
103 else
104 cat
105 fi
106 } >>"$_file"
107 ;;
108 *)
109 # Handle all syslog:* variants here too. There's no tool to do
110 # the lossy things, so just use logger.
111 logger -t "ctdbd: ${_tag}" $*
112 ;;
113 esac
114 }
116 # When things are run in the background in an eventscript then logging
117 # output might get lost. This is the "solution". :-)
118 background_with_logging ()
119 {
120 (
121 "$@" 2>&1 </dev/null |
122 script_log "${script_name}&"
123 )&
125 return 0
126 }
128 ##############################################################
129 # check number of args for different events
130 ctdb_check_args ()
131 {
132 case "$1" in
133 takeip|releaseip)
134 if [ $# != 4 ]; then
135 echo "ERROR: must supply interface, IP and maskbits"
136 exit 1
137 fi
138 ;;
139 updateip)
140 if [ $# != 5 ]; then
141 echo "ERROR: must supply old interface, new interface, IP and maskbits"
142 exit 1
143 fi
144 ;;
145 esac
146 }
148 ##############################################################
149 # determine on what type of system (init style) we are running
150 detect_init_style()
151 {
152 # only do detection if not already set:
153 [ -z "$CTDB_INIT_STYLE" ] || return
155 if [ -x /sbin/startproc ]; then
156 CTDB_INIT_STYLE="suse"
157 elif [ -x /sbin/start-stop-daemon ]; then
158 CTDB_INIT_STYLE="debian"
159 else
160 CTDB_INIT_STYLE="redhat"
161 fi
162 }
164 ######################################################
165 # simulate /sbin/service on platforms that don't have it
166 # _service() makes it easier to hook the service() function for
167 # testing.
168 _service ()
169 {
170 _service_name="$1"
171 _op="$2"
173 # do nothing, when no service was specified
174 [ -z "$_service_name" ] && return
176 if [ -x /sbin/service ]; then
177 $_nice /sbin/service "$_service_name" "$_op"
178 elif [ -x /usr/sbin/service ]; then
179 $_nice /usr/sbin/service "$_service_name" "$_op"
180 elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
181 $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
182 elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
183 $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
184 fi
185 }
187 service()
188 {
189 _nice=""
190 _service "$@"
191 }
193 ######################################################
194 # simulate /sbin/service (niced) on platforms that don't have it
195 nice_service()
196 {
197 _nice="nice"
198 _service "$@"
199 }
201 ######################################################
202 # Cached retrieval of PNN from local node. This never changes so why
203 # open a client connection to the server each time this is needed?
204 # This sets $pnn - this avoid an unnecessary subprocess.
205 ctdb_get_pnn ()
206 {
207 _pnn_file="$CTDB_VARDIR/state/my-pnn"
208 if [ ! -f "$_pnn_file" ] ; then
209 ctdb pnn | sed -e 's@.*:@@' >"$_pnn_file"
210 fi
212 read pnn <"$_pnn_file"
213 }
215 ######################################################
216 # wrapper around /proc/ settings to allow them to be hooked
217 # for testing
218 # 1st arg is relative path under /proc/, 2nd arg is value to set
219 set_proc ()
220 {
221 echo "$2" >"/proc/$1"
222 }
224 set_proc_maybe ()
225 {
226 if [ -w "/proc/$1" ] ; then
227 set_proc "$1" "$2"
228 fi
229 }
231 ######################################################
232 # wrapper around getting file contents from /proc/ to allow
233 # this to be hooked for testing
234 # 1st arg is relative path under /proc/
235 get_proc ()
236 {
237 cat "/proc/$1"
238 }
240 ######################################################
241 # Print up to $_max kernel stack traces for processes named $_program
242 program_stack_traces ()
243 {
244 _prog="$1"
245 _max="${2:-1}"
247 _count=1
248 for _pid in $(pidof "$_prog") ; do
249 [ $_count -le $_max ] || break
251 # Do this first to avoid racing with process exit
252 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
253 if [ -n "$_stack" ] ; then
254 echo "Stack trace for ${_prog}[${_pid}]:"
255 echo "$_stack"
256 _count=$(($_count + 1))
257 fi
258 done
259 }
261 ######################################################
262 # Check that an RPC service is healthy -
263 # this includes allowing a certain number of failures
264 # before marking the NFS service unhealthy.
265 #
266 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
267 #
268 # each triple is a set of 3 arguments: an operator, a
269 # fail count limit and an action string.
270 #
271 # For example:
272 #
273 # nfs_check_rpc_service "lockd" \
274 # -ge 15 "verbose restart unhealthy" \
275 # -eq 10 "restart:bs"
276 #
277 # says that if lockd is down for 15 iterations then do
278 # a verbose restart of lockd and mark the node unhealthy.
279 # Before this, after 10 iterations of failure, the
280 # service is restarted silently in the background.
281 # Order is important: the number of failures need to be
282 # specified in reverse order because processing stops
283 # after the first condition that is true.
284 ######################################################
285 nfs_check_rpc_service ()
286 {
287 _prog_name="$1" ; shift
289 if _nfs_check_rpc_common "$_prog_name" ; then
290 return
291 fi
293 while [ -n "$3" ] ; do
294 if _nfs_check_rpc_action "$1" "$2" "$3" ; then
295 break
296 fi
297 shift 3
298 done
299 }
301 # The new way of doing things...
302 nfs_check_rpc_services ()
303 {
304 # Files must end with .check - avoids editor backups, RPM fu, ...
305 for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
306 _t="${_f%.check}"
307 _prog_name="${_t##*/[0-9][0-9].}"
309 # If $_prog_name contains '@' then the bit after it is the
310 # address family.
311 _family="${_prog_name#*@}"
312 if [ "$_family" = "$_prog_name" ] ; then
313 _family=""
314 else
315 _prog_name="${_prog_name%@*}"
316 fi
318 if _nfs_check_rpc_common "$_prog_name" "$_family" ; then
319 # This RPC service is up, check next service...
320 continue
321 fi
323 # Check each line in the file in turn until one of the limit
324 # checks is hit...
325 while read _cmp _lim _rest ; do
326 # Skip comments
327 case "$_cmp" in
328 \#*) continue ;;
329 esac
331 if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
332 # Limit was hit on this line, no further checking...
333 break
334 fi
335 done <"$_f"
336 done
337 }
339 _nfs_check_rpc_common ()
340 {
341 _prog_name="$1"
342 _family="$2"
344 # Some platforms don't have separate programs for all services.
345 case "$_prog_name" in
346 statd)
347 type "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
348 esac
350 case "$_prog_name" in
351 nfsd)
352 _rpc_prog=nfs
353 _version=3
354 ;;
355 mountd)
356 _rpc_prog=mountd
357 _version=1
358 ;;
359 rquotad)
360 _rpc_prog=rquotad
361 _version=1
362 ;;
363 lockd)
364 _rpc_prog=nlockmgr
365 _version=4
366 ;;
367 statd)
368 _rpc_prog=status
369 _version=1
370 ;;
371 *)
372 echo "Internal error: unknown RPC program \"$_prog_name\"."
373 exit 1
374 esac
376 _service_name="nfs_${_prog_name}${_family:+_}${_family}"
378 if ctdb_check_rpc "$_rpc_prog" "$_version" "$_family" >/dev/null ; then
379 ctdb_counter_init "$_service_name"
380 return 0
381 fi
383 ctdb_counter_incr "$_service_name"
385 return 1
386 }
388 _nfs_check_rpc_action ()
389 {
390 _cmp="$1"
391 _limit="$2"
392 _actions="$3"
394 if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
395 return 1
396 fi
398 for _action in $_actions ; do
399 case "$_action" in
400 verbose)
401 echo "$ctdb_check_rpc_out"
402 ;;
403 restart)
404 _nfs_restart_rpc_service "$_prog_name"
405 ;;
406 restart:b)
407 _nfs_restart_rpc_service "$_prog_name" true
408 ;;
409 unhealthy)
410 exit 1
411 ;;
412 *)
413 echo "Internal error: unknown action \"$_action\"."
414 exit 1
415 esac
416 done
418 return 0
419 }
421 _nfs_restart_rpc_service ()
422 {
423 _prog_name="$1"
424 _background="${2:-false}"
426 if $_background ; then
427 _maybe_background="background_with_logging"
428 else
429 _maybe_background=""
430 fi
432 _p="rpc.${_prog_name}"
434 case "$_prog_name" in
435 nfsd)
436 echo "Trying to restart NFS service"
437 $_maybe_background startstop_nfs restart
438 ;;
439 mountd)
440 echo "Trying to restart $_prog_name [${_p}]"
441 killall -q -9 "$_p"
442 nfs_dump_some_threads "$_p"
443 $_maybe_background $_p $RPCMOUNTDOPTS \
444 ${MOUNTD_PORT:+-p} $MOUNTD_PORT
445 ;;
446 rquotad)
447 echo "Trying to restart $_prog_name [${_p}]"
448 killall -q -9 "$_p"
449 nfs_dump_some_threads "$_p"
450 $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
451 ;;
452 lockd)
453 echo "Trying to restart lock manager service"
454 $_maybe_background startstop_nfslock restart
455 ;;
456 statd)
457 echo "Trying to restart $_prog_name [${_p}]"
458 killall -q -9 "$_p"
459 nfs_dump_some_threads "$_p"
460 $_maybe_background $_p \
461 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
462 ${STATD_PORT:+-p} $STATD_PORT \
463 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
464 ;;
465 *)
466 echo "Internal error: unknown RPC program \"$_prog_name\"."
467 exit 1
468 esac
469 }
471 ######################################################
472 # Check an RPC service with rpcinfo
473 ######################################################
474 ctdb_check_rpc ()
475 {
476 _progname="$1" # passed to rpcinfo (looked up in /etc/rpc)
477 _version="$2" # optional, not passed if empty/unset
478 _family="${3:-tcp}" # optional, default is "tcp"
480 _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
482 if ! ctdb_check_rpc_out=$(rpcinfo -T $_family $_localhost \
483 $_progname $_version 2>&1) ; then
484 ctdb_check_rpc_out="ERROR: $_progname failed RPC check:
485 $ctdb_check_rpc_out"
486 echo "$ctdb_check_rpc_out"
487 return 1
488 fi
489 }
491 ######################################################
492 # Ensure $service_name is set
493 assert_service_name ()
494 {
495 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
496 }
498 ######################################################
499 # check a set of directories is available
500 # return 1 on a missing directory
501 # directories are read from stdin
502 ######################################################
503 ctdb_check_directories_probe()
504 {
505 while IFS="" read d ; do
506 case "$d" in
507 *%*)
508 continue
509 ;;
510 *)
511 [ -d "${d}/." ] || return 1
512 esac
513 done
514 }
516 ######################################################
517 # check a set of directories is available
518 # directories are read from stdin
519 ######################################################
520 ctdb_check_directories()
521 {
522 ctdb_check_directories_probe || {
523 echo "ERROR: $service_name directory \"$d\" not available"
524 exit 1
525 }
526 }
528 ######################################################
529 # check a set of tcp ports
530 # usage: ctdb_check_tcp_ports <ports...>
531 ######################################################
533 # This flag file is created when a service is initially started. It
534 # is deleted the first time TCP port checks for that service succeed.
535 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
536 # message if a port check fails.
537 _ctdb_check_tcp_common ()
538 {
539 assert_service_name
540 _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
541 }
543 ctdb_check_tcp_init ()
544 {
545 _ctdb_check_tcp_common
546 mkdir -p "${_ctdb_service_started_file%/*}" # dirname
547 touch "$_ctdb_service_started_file"
548 }
550 # Check whether something is listening on all of the given TCP ports
551 # using the "ctdb checktcpport" command.
552 ctdb_check_tcp_ports()
553 {
554 if [ -z "$1" ] ; then
555 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
556 exit 1
557 fi
559 for _p ; do # process each function argument (port)
560 _cmd="ctdb checktcpport $_p"
561 _out=$($_cmd 2>&1)
562 _ret=$?
563 case "$_ret" in
564 0)
565 _ctdb_check_tcp_common
566 if [ ! -f "$_ctdb_service_started_file" ] ; then
567 echo "ERROR: $service_name tcp port $_p is not responding"
568 debug "\"ctdb checktcpport $_p\" was able to bind to port"
569 else
570 echo "INFO: $service_name tcp port $_p is not responding"
571 fi
573 return 1
574 ;;
575 98)
576 # Couldn't bind, something already listening, next port...
577 continue
578 ;;
579 *)
580 echo "ERROR: unexpected error running \"ctdb checktcpport\""
581 debug <<EOF
582 ctdb checktcpport (exited with $_ret) with output:
583 $_out"
584 EOF
585 return $_ret
586 esac
587 done
589 # All ports listening
590 _ctdb_check_tcp_common
591 rm -f "$_ctdb_service_started_file"
592 return 0
593 }
595 ######################################################
596 # check a unix socket
597 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
598 ######################################################
599 ctdb_check_unix_socket() {
600 socket_path="$1"
601 [ -z "$socket_path" ] && return
603 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
604 echo "ERROR: $service_name socket $socket_path not found"
605 return 1
606 fi
607 }
609 ######################################################
610 # check a command returns zero status
611 # usage: ctdb_check_command <command>
612 ######################################################
613 ctdb_check_command ()
614 {
615 _out=$("$@" 2>&1) || {
616 echo "ERROR: $* returned error"
617 echo "$_out" | debug
618 exit 1
619 }
620 }
622 ################################################
623 # kill off any TCP connections with the given IP
624 ################################################
625 kill_tcp_connections ()
626 {
627 _ip="$1"
629 _oneway=false
630 if [ "$2" = "oneway" ] ; then
631 _oneway=true
632 fi
634 get_tcp_connections_for_ip "$_ip" | {
635 _killcount=0
636 _connections=""
637 _nl="
638 "
639 while read _dst _src; do
640 _destport="${_dst##*:}"
641 __oneway=$_oneway
642 case $_destport in
643 # we only do one-way killtcp for CIFS
644 139|445) __oneway=true ;;
645 esac
647 echo "Killing TCP connection $_src $_dst"
648 _connections="${_connections}${_nl}${_src} ${_dst}"
649 if ! $__oneway ; then
650 _connections="${_connections}${_nl}${_dst} ${_src}"
651 fi
653 _killcount=$(($_killcount + 1))
654 done
656 if [ $_killcount -eq 0 ] ; then
657 return
658 fi
660 echo "$_connections" | ctdb killtcp || {
661 echo "Failed to send killtcp control"
662 return
663 }
665 _count=0
666 while : ; do
667 _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
669 if [ $_remaining -eq 0 ] ; then
670 echo "Killed $_killcount TCP connections to released IP $_ip"
671 return
672 fi
674 _count=$(($_count + 1))
675 if [ $_count -gt 3 ] ; then
676 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
677 return
678 fi
680 echo "Waiting for $_remaining connections to be killed for IP $_ip"
681 sleep 1
682 done
683 }
684 }
686 ##################################################################
687 # kill off the local end for any TCP connections with the given IP
688 ##################################################################
689 kill_tcp_connections_local_only ()
690 {
691 kill_tcp_connections "$1" "oneway"
692 }
694 ##################################################################
695 # tickle any TCP connections with the given IP
696 ##################################################################
697 tickle_tcp_connections ()
698 {
699 _ip="$1"
701 get_tcp_connections_for_ip "$_ip" |
702 {
703 _failed=false
705 while read dest src; do
706 echo "Tickle TCP connection $src $dest"
707 ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
708 echo "Tickle TCP connection $dest $src"
709 ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
710 done
712 if $_failed ; then
713 echo "Failed to send tickle control"
714 fi
715 }
716 }
718 get_tcp_connections_for_ip ()
719 {
720 _ip="$1"
722 netstat -tn | awk -v ip=$_ip \
723 'index($1, "tcp") == 1 && \
724 (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
725 && $6 == "ESTABLISHED" \
726 {print $4" "$5}'
727 }
729 ##################################################################
730 # use statd-callout to update NFS lock info
731 ##################################################################
732 nfs_update_lock_info ()
733 {
734 if [ -x "$CTDB_BASE/statd-callout" ] ; then
735 "$CTDB_BASE/statd-callout" update
736 fi
737 }
739 ########################################################
740 # start/stop the Ganesha nfs service
741 ########################################################
742 startstop_ganesha()
743 {
744 _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
745 case "$1" in
746 start)
747 service "$_service_name" start
748 ;;
749 stop)
750 service "$_service_name" stop
751 ;;
752 restart)
753 service "$_service_name" stop
754 nfs_dump_some_threads "rpc.statd"
755 service "$_service_name" start
756 ;;
757 esac
758 }
760 ########################################################
761 # start/stop the nfs service on different platforms
762 ########################################################
763 startstop_nfs() {
764 PLATFORM="unknown"
765 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
766 PLATFORM="sles"
767 }
768 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
769 -r /usr/lib/systemd/system/nfs-lock.service ] && {
770 PLATFORM="rhel"
771 }
773 case $PLATFORM in
774 sles)
775 case $1 in
776 start)
777 service nfsserver start
778 ;;
779 stop)
780 service nfsserver stop > /dev/null 2>&1
781 ;;
782 restart)
783 set_proc "fs/nfsd/threads" 0
784 service nfsserver stop > /dev/null 2>&1
785 pkill -9 nfsd
786 nfs_dump_some_threads
787 service nfsserver start
788 ;;
789 esac
790 ;;
791 rhel)
792 case $1 in
793 start)
794 service nfslock start
795 service nfs start
796 ;;
797 stop)
798 service nfs stop
799 service nfslock stop
800 ;;
801 restart)
802 set_proc "fs/nfsd/threads" 0
803 service nfs stop > /dev/null 2>&1
804 service nfslock stop > /dev/null 2>&1
805 pkill -9 nfsd
806 nfs_dump_some_threads
807 service nfslock start
808 service nfs start
809 ;;
810 esac
811 ;;
812 *)
813 echo "Unknown platform. NFS is not supported with ctdb"
814 exit 1
815 ;;
816 esac
817 }
819 # Dump up to the configured number of nfsd thread backtraces.
820 nfs_dump_some_threads ()
821 {
822 _prog="${1:-nfsd}"
824 _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
825 [ $_num -gt 0 ] || return 0
827 program_stack_traces "$_prog" $_num
828 }
830 ########################################################
831 # start/stop the nfs lockmanager service on different platforms
832 ########################################################
833 startstop_nfslock() {
834 PLATFORM="unknown"
835 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
836 PLATFORM="sles"
837 }
838 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
839 -r /usr/lib/systemd/system/nfs-lock.service ] && {
840 PLATFORM="rhel"
841 }
843 case $PLATFORM in
844 sles)
845 # for sles there is no service for lockmanager
846 # so we instead just shutdown/restart nfs
847 case $1 in
848 start)
849 service nfsserver start
850 ;;
851 stop)
852 service nfsserver stop > /dev/null 2>&1
853 ;;
854 restart)
855 service nfsserver stop > /dev/null 2>&1
856 service nfsserver start
857 ;;
858 esac
859 ;;
860 rhel)
861 case $1 in
862 start)
863 service nfslock start
864 ;;
865 stop)
866 service nfslock stop > /dev/null 2>&1
867 ;;
868 restart)
869 service nfslock stop > /dev/null 2>&1
870 service nfslock start
871 ;;
872 esac
873 ;;
874 *)
875 echo "Unknown platform. NFS locking is not supported with ctdb"
876 exit 1
877 ;;
878 esac
879 }
881 ########################################################
883 add_ip_to_iface ()
884 {
885 _iface=$1
886 _ip=$2
887 _maskbits=$3
889 # Ensure interface is up
890 ip link set "$_iface" up || \
891 die "Failed to bringup interface $_iface"
893 # Only need to define broadcast for IPv4
894 case "$ip" in
895 *:*) _bcast="" ;;
896 *) _bcast="brd +" ;;
897 esac
899 ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
900 echo "Failed to add $_ip/$_maskbits on dev $_iface"
901 return 1
902 }
904 # Wait 5 seconds for IPv6 addresses to stop being tentative...
905 if [ -z "$_bcast" ] ; then
906 for _x in $(seq 1 10) ; do
907 ip addr show to "${_ip}/128" | grep -q "tentative" || break
908 sleep 0.5
909 done
911 # If the address was a duplicate then it won't be on the
912 # interface so flag an error.
913 _t=$(ip addr show to "${_ip}/128")
914 case "$_t" in
915 "")
916 echo "Failed to add $_ip/$_maskbits on dev $_iface"
917 return 1
918 ;;
919 *tentative*|*dadfailed*)
920 echo "Failed to add $_ip/$_maskbits on dev $_iface"
921 ip addr del "$_ip/$_maskbits" dev "$_iface"
922 return 1
923 ;;
924 esac
925 fi
926 }
928 delete_ip_from_iface()
929 {
930 _iface=$1
931 _ip=$2
932 _maskbits=$3
934 # This could be set globally for all interfaces but it is probably
935 # better to avoid surprises, so limit it the interfaces where CTDB
936 # has public IP addresses. There isn't anywhere else convenient
937 # to do this so just set it each time. This is much cheaper than
938 # remembering and re-adding secondaries.
939 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
941 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
942 echo "Failed to del $_ip on dev $_iface"
943 return 1
944 }
945 }
947 # If the given IP is hosted then print 2 items: maskbits and iface
948 ip_maskbits_iface ()
949 {
950 _addr="$1"
952 case "$_addr" in
953 *:*) _family="inet6" ; _bits=128 ;;
954 *) _family="inet" ; _bits=32 ;;
955 esac
957 ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
958 awk -v family="${_family}" \
959 'NR == 1 { iface = $2; sub(":$", "", iface) ; \
960 sub("@.*", "", iface) } \
961 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
962 print mask, iface, family }'
963 }
965 drop_ip ()
966 {
967 _addr="${1%/*}" # Remove optional maskbits
969 set -- $(ip_maskbits_iface $_addr)
970 if [ -n "$1" ] ; then
971 _maskbits="$1"
972 _iface="$2"
973 echo "Removing public address $_addr/$_maskbits from device $_iface"
974 delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
975 fi
976 }
978 drop_all_public_ips ()
979 {
980 while read _ip _x ; do
981 drop_ip "$_ip"
982 done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
983 }
985 flush_route_cache ()
986 {
987 set_proc_maybe sys/net/ipv4/route/flush 1
988 set_proc_maybe sys/net/ipv6/route/flush 1
989 }
991 ########################################################
992 # Simple counters
993 _ctdb_counter_common () {
994 _service_name="${1:-${service_name:-${script_name}}}"
995 _counter_file="$ctdb_fail_dir/$_service_name"
996 mkdir -p "${_counter_file%/*}" # dirname
997 }
998 ctdb_counter_init () {
999 _ctdb_counter_common "$1"
1001 >"$_counter_file"
1002 }
1003 ctdb_counter_incr () {
1004 _ctdb_counter_common "$1"
1006 # unary counting!
1007 echo -n 1 >> "$_counter_file"
1008 }
1009 ctdb_check_counter () {
1010 _msg="${1:-error}" # "error" - anything else is silent on fail
1011 _op="${2:--ge}" # an integer operator supported by test
1012 _limit="${3:-${service_fail_limit}}"
1013 shift 3
1014 _ctdb_counter_common "$1"
1016 # unary counting!
1017 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
1018 _hit=false
1019 if [ "$_op" != "%" ] ; then
1020 if [ $_size $_op $_limit ] ; then
1021 _hit=true
1022 fi
1023 else
1024 if [ $(($_size $_op $_limit)) -eq 0 ] ; then
1025 _hit=true
1026 fi
1027 fi
1028 if $_hit ; then
1029 if [ "$_msg" = "error" ] ; then
1030 echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
1031 exit 1
1032 else
1033 return 1
1034 fi
1035 fi
1036 }
1038 ########################################################
1040 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1041 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1043 ctdb_setup_service_state_dir ()
1044 {
1045 service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1046 mkdir -p "$service_state_dir" || {
1047 echo "Error creating state dir \"$service_state_dir\""
1048 exit 1
1049 }
1050 }
1052 ########################################################
1053 # Managed status history, for auto-start/stop
1055 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1057 _ctdb_managed_common ()
1058 {
1059 _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1060 }
1062 ctdb_service_managed ()
1063 {
1064 _ctdb_managed_common
1065 mkdir -p "$ctdb_managed_dir"
1066 touch "$_ctdb_managed_file"
1067 }
1069 ctdb_service_unmanaged ()
1070 {
1071 _ctdb_managed_common
1072 rm -f "$_ctdb_managed_file"
1073 }
1075 is_ctdb_previously_managed_service ()
1076 {
1077 _ctdb_managed_common
1078 [ -f "$_ctdb_managed_file" ]
1079 }
1081 ########################################################
1082 # Check and set status
1084 log_status_cat ()
1085 {
1086 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1087 }
1089 ctdb_checkstatus ()
1090 {
1091 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1092 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1093 return 1
1094 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1095 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1096 return 2
1097 else
1098 return 0
1099 fi
1100 }
1102 ctdb_setstatus ()
1103 {
1104 d="$ctdb_status_dir/$script_name"
1105 case "$1" in
1106 unhealthy|banned)
1107 mkdir -p "$d"
1108 cat "$2" >"$d/$1"
1109 ;;
1110 *)
1111 for i in "banned" "unhealthy" ; do
1112 rm -f "$d/$i"
1113 done
1114 ;;
1115 esac
1116 }
1118 ##################################################################
1119 # Reconfigure a service on demand
1121 _ctdb_service_reconfigure_common ()
1122 {
1123 _d="$ctdb_status_dir/${service_name}"
1124 mkdir -p "$_d"
1125 _ctdb_service_reconfigure_flag="$_d/reconfigure"
1126 }
1128 ctdb_service_needs_reconfigure ()
1129 {
1130 _ctdb_service_reconfigure_common
1131 [ -e "$_ctdb_service_reconfigure_flag" ]
1132 }
1134 ctdb_service_set_reconfigure ()
1135 {
1136 _ctdb_service_reconfigure_common
1137 >"$_ctdb_service_reconfigure_flag"
1138 }
1140 ctdb_service_unset_reconfigure ()
1141 {
1142 _ctdb_service_reconfigure_common
1143 rm -f "$_ctdb_service_reconfigure_flag"
1144 }
1146 ctdb_service_reconfigure ()
1147 {
1148 echo "Reconfiguring service \"${service_name}\"..."
1149 ctdb_service_unset_reconfigure
1150 service_reconfigure || return $?
1151 ctdb_counter_init
1152 }
1154 # Default service_reconfigure() function does nothing.
1155 service_reconfigure ()
1156 {
1157 :
1158 }
1160 ctdb_reconfigure_take_lock ()
1161 {
1162 _ctdb_service_reconfigure_common
1163 _lock="${_d}/reconfigure_lock"
1164 mkdir -p "${_lock%/*}" # dirname
1165 touch "$_lock"
1167 (
1168 flock 0
1169 # This is overkill but will work if we need to extend this to
1170 # allow certain events to run multiple times in parallel
1171 # (e.g. takeip) and write multiple PIDs to the file.
1172 read _locker_event
1173 if [ -n "$_locker_event" ] ; then
1174 while read _pid ; do
1175 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1176 kill -0 "$_pid" 2>/dev/null ; then
1177 exit 1
1178 fi
1179 done
1180 fi
1182 printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1183 exit 0
1184 ) <"$_lock"
1185 }
1187 ctdb_reconfigure_release_lock ()
1188 {
1189 _ctdb_service_reconfigure_common
1190 _lock="${_d}/reconfigure_lock"
1192 rm -f "$_lock"
1193 }
1195 ctdb_replay_monitor_status ()
1196 {
1197 echo "Replaying previous status for this script due to reconfigure..."
1198 # Leading separator ('|') is missing in some versions...
1199 _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1200 # Output looks like this:
1201 # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1202 # This is the cheapest way of getting fields in the middle.
1203 set -- $(IFS="|" ; echo $_out)
1204 _code="$3"
1205 _status="$4"
1206 # The error output field can include colons so we'll try to
1207 # preserve them. The weak checking at the beginning tries to make
1208 # this work for both broken (no leading '|') and fixed output.
1209 _out="${_out%|}"
1210 _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1211 case "$_status" in
1212 OK) : ;; # Do nothing special.
1213 TIMEDOUT)
1214 # Recast this as an error, since we can't exit with the
1215 # correct negative number.
1216 _code=1
1217 _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1218 ;;
1219 DISABLED)
1220 # Recast this as an OK, since we can't exit with the
1221 # correct negative number.
1222 _code=0
1223 _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1224 ;;
1225 *) : ;; # Must be ERROR, do nothing special.
1226 esac
1227 if [ -n "$_err_out" ] ; then
1228 echo "$_err_out"
1229 fi
1230 exit $_code
1231 }
1233 ctdb_service_check_reconfigure ()
1234 {
1235 assert_service_name
1237 # We only care about some events in this function. For others we
1238 # return now.
1239 case "$event_name" in
1240 monitor|ipreallocated|reconfigure) : ;;
1241 *) return 0 ;;
1242 esac
1244 if ctdb_reconfigure_take_lock ; then
1245 # No events covered by this function are running, so proceed
1246 # with gay abandon.
1247 case "$event_name" in
1248 reconfigure)
1249 (ctdb_service_reconfigure)
1250 exit $?
1251 ;;
1252 ipreallocated)
1253 if ctdb_service_needs_reconfigure ; then
1254 ctdb_service_reconfigure
1255 fi
1256 ;;
1257 esac
1259 ctdb_reconfigure_release_lock
1260 else
1261 # Somebody else is running an event we don't want to collide
1262 # with. We proceed with caution.
1263 case "$event_name" in
1264 reconfigure)
1265 # Tell whoever called us to retry.
1266 exit 2
1267 ;;
1268 ipreallocated)
1269 # Defer any scheduled reconfigure and just run the
1270 # rest of the ipreallocated event, as per the
1271 # eventscript. There's an assumption here that the
1272 # event doesn't depend on any scheduled reconfigure.
1273 # This is true in the current code.
1274 return 0
1275 ;;
1276 monitor)
1277 # There is most likely a reconfigure in progress so
1278 # the service is possibly unstable. As above, we
1279 # defer any scheduled reconfigured. We also replay
1280 # the previous monitor status since that's the best
1281 # information we have.
1282 ctdb_replay_monitor_status
1283 ;;
1284 esac
1285 fi
1286 }
1288 ##################################################################
1289 # Does CTDB manage this service? - and associated auto-start/stop
1291 ctdb_compat_managed_service ()
1292 {
1293 if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1294 CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1295 fi
1296 }
1298 is_ctdb_managed_service ()
1299 {
1300 assert_service_name
1302 # $t is used just for readability and to allow better accurate
1303 # matching via leading/trailing spaces
1304 t=" $CTDB_MANAGED_SERVICES "
1306 # Return 0 if "<space>$service_name<space>" appears in $t
1307 if [ "${t#* ${service_name} }" != "${t}" ] ; then
1308 return 0
1309 fi
1311 # If above didn't match then update $CTDB_MANAGED_SERVICES for
1312 # backward compatibility and try again.
1313 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
1314 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
1315 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
1316 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "apache2"
1317 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
1318 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
1319 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
1320 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
1321 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
1323 t=" $CTDB_MANAGED_SERVICES "
1325 # Return 0 if "<space>$service_name<space>" appears in $t
1326 [ "${t#* ${service_name} }" != "${t}" ]
1327 }
1329 ctdb_start_stop_service ()
1330 {
1331 assert_service_name
1333 # Allow service-start/service-stop pseudo-events to start/stop
1334 # services when we're not auto-starting/stopping and we're not
1335 # monitoring.
1336 case "$event_name" in
1337 service-start)
1338 if is_ctdb_managed_service ; then
1339 die 'service-start event not permitted when service is managed'
1340 fi
1341 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1342 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1343 fi
1344 ctdb_service_start
1345 exit $?
1346 ;;
1347 service-stop)
1348 if is_ctdb_managed_service ; then
1349 die 'service-stop event not permitted when service is managed'
1350 fi
1351 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1352 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1353 fi
1354 ctdb_service_stop
1355 exit $?
1356 ;;
1357 esac
1359 # Do nothing unless configured to...
1360 [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1362 [ "$event_name" = "monitor" ] || return 0
1364 if is_ctdb_managed_service ; then
1365 if ! is_ctdb_previously_managed_service ; then
1366 echo "Starting service \"$service_name\" - now managed"
1367 background_with_logging ctdb_service_start
1368 exit $?
1369 fi
1370 else
1371 if is_ctdb_previously_managed_service ; then
1372 echo "Stopping service \"$service_name\" - no longer managed"
1373 background_with_logging ctdb_service_stop
1374 exit $?
1375 fi
1376 fi
1377 }
1379 ctdb_service_start ()
1380 {
1381 # The service is marked managed if we've ever tried to start it.
1382 ctdb_service_managed
1384 service_start || return $?
1386 ctdb_counter_init
1387 ctdb_check_tcp_init
1388 }
1390 ctdb_service_stop ()
1391 {
1392 ctdb_service_unmanaged
1393 service_stop
1394 }
1396 # Default service_start() and service_stop() functions.
1398 # These may be overridden in an eventscript.
1399 service_start ()
1400 {
1401 service "$service_name" start
1402 }
1404 service_stop ()
1405 {
1406 service "$service_name" stop
1407 }
1409 ##################################################################
1411 ctdb_standard_event_handler ()
1412 {
1413 case "$1" in
1414 status)
1415 ctdb_checkstatus
1416 exit
1417 ;;
1418 setstatus)
1419 shift
1420 ctdb_setstatus "$@"
1421 exit
1422 ;;
1423 esac
1424 }
1426 iptables_wrapper ()
1427 {
1428 _family="$1" ; shift
1429 if [ "$_family" = "inet6" ] ; then
1430 _iptables_cmd="ip6tables"
1431 else
1432 _iptables_cmd="iptables"
1433 fi
1435 # iptables doesn't like being re-entered, so flock-wrap it.
1436 flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1437 }
1439 # AIX (and perhaps others?) doesn't have mktemp
1440 if ! type mktemp >/dev/null 2>&1 ; then
1441 mktemp ()
1442 {
1443 _dir=false
1444 if [ "$1" = "-d" ] ; then
1445 _dir=true
1446 shift
1447 fi
1448 _d="${TMPDIR:-/tmp}"
1449 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1450 md5sum | \
1451 sed -e 's@\(..........\).*@\1@')
1452 _t="${_d}/tmp.${_hex10}"
1453 (
1454 umask 077
1455 if $_dir ; then
1456 mkdir "$_t"
1457 else
1458 >"$_t"
1459 fi
1460 )
1461 echo "$_t"
1462 }
1463 fi
1465 ########################################################
1466 # tickle handling
1467 ########################################################
1469 update_tickles ()
1470 {
1471 _port="$1"
1473 tickledir="$CTDB_VARDIR/state/tickles"
1474 mkdir -p "$tickledir"
1476 ctdb_get_pnn
1478 # What public IPs do I hold?
1479 _ips=$(ctdb -X ip | awk -F'|' -v pnn=$pnn '$3 == pnn {print $2}')
1481 # IPs as a regexp choice
1482 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1484 # Record connections to our public IPs in a temporary file
1485 _my_connections="${tickledir}/${_port}.connections"
1486 rm -f "$_my_connections"
1487 netstat -tn |
1488 awk -v destpat="^${_ipschoice}:${_port}\$" \
1489 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1490 sort >"$_my_connections"
1492 # Record our current tickles in a temporary file
1493 _my_tickles="${tickledir}/${_port}.tickles"
1494 rm -f "$_my_tickles"
1495 for _i in $_ips ; do
1496 ctdb -X gettickles $_i $_port |
1497 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1498 done |
1499 sort >"$_my_tickles"
1501 # Add tickles for connections that we haven't already got tickles for
1502 comm -23 "$_my_connections" "$_my_tickles" |
1503 while read _src _dst ; do
1504 ctdb addtickle $_src $_dst
1505 done
1507 # Remove tickles for connections that are no longer there
1508 comm -13 "$_my_connections" "$_my_tickles" |
1509 while read _src _dst ; do
1510 ctdb deltickle $_src $_dst
1511 done
1513 rm -f "$_my_connections" "$_my_tickles"
1514 }
1516 ########################################################
1517 # load a site local config file
1518 ########################################################
1520 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1521 . "$CTDB_RC_LOCAL"
1522 }
1524 [ -x $CTDB_BASE/rc.local ] && {
1525 . $CTDB_BASE/rc.local
1526 }
1528 [ -d $CTDB_BASE/rc.local.d ] && {
1529 for i in $CTDB_BASE/rc.local.d/* ; do
1530 [ -x "$i" ] && . "$i"
1531 done
1532 }
1534 script_name="${0##*/}" # basename
1535 service_fail_limit=1
1536 event_name="$1"