| blob | e6fe43f1e8aafeaada2cd0443ac896566c9a320a |
1 # Hey Emacs, this is a -*- shell-script -*- !!!
3 # utility functions for ctdb event scripts
5 [ -z "$CTDB_VARDIR" ] && {
6 if [ -d "/var/lib/ctdb" ] ; then
7 export CTDB_VARDIR="/var/lib/ctdb"
8 else
9 export CTDB_VARDIR="/var/ctdb"
10 fi
11 }
12 [ -z "$CTDB_ETCDIR" ] && {
13 export CTDB_ETCDIR="/etc"
14 }
16 #######################################
17 # pull in a system config file, if any
18 _loadconfig() {
20 if [ -z "$1" ] ; then
21 foo="${service_config:-${service_name}}"
22 if [ -n "$foo" ] ; then
23 loadconfig "$foo"
24 return
25 fi
26 fi
28 if [ "$1" != "ctdb" ] ; then
29 loadconfig "ctdb"
30 fi
32 if [ -z "$1" ] ; then
33 return
34 fi
36 if [ -f $CTDB_ETCDIR/sysconfig/$1 ]; then
37 . $CTDB_ETCDIR/sysconfig/$1
38 elif [ -f $CTDB_ETCDIR/default/$1 ]; then
39 . $CTDB_ETCDIR/default/$1
40 elif [ -f $CTDB_BASE/sysconfig/$1 ]; then
41 . $CTDB_BASE/sysconfig/$1
42 fi
44 if [ "$1" = "ctdb" ] ; then
45 _config="${CTDB_BASE}/ctdbd.conf"
46 if [ -r "$_config" ] ; then
47 . "$_config"
48 fi
49 fi
50 }
52 loadconfig () {
53 _loadconfig "$@"
54 }
56 ##############################################################
58 # CTDB_SCRIPT_DEBUGLEVEL can be overwritten by setting it in a
59 # configuration file.
60 debug ()
61 {
62 if [ ${CTDB_SCRIPT_DEBUGLEVEL:-2} -ge 4 ] ; then
63 # If there are arguments then echo them. Otherwise expect to
64 # use stdin, which allows us to pass lots of debug using a
65 # here document.
66 if [ -n "$1" ] ; then
67 echo "DEBUG: $*"
68 elif ! tty -s ; then
69 sed -e 's@^@DEBUG: @'
70 fi
71 fi
72 }
74 die ()
75 {
76 _msg="$1"
77 _rc="${2:-1}"
79 echo "$_msg"
80 exit $_rc
81 }
83 # Log given message or stdin to either syslog or a CTDB log file
84 # $1 is the tag passed to logger if syslog is in use.
85 script_log ()
86 {
87 _tag="$1" ; shift
89 case "$CTDB_LOGGING" in
90 file:*|"")
91 if [ -n "$CTDB_LOGGING" ] ; then
92 _file="${CTDB_LOGGING#file:}"
93 else
94 _file="/var/log/log.ctdb"
95 fi
96 {
97 if [ -n "$*" ] ; then
98 echo "$*"
99 else
100 cat
101 fi
102 } >>"$_file"
103 ;;
104 *)
105 # Handle all syslog:* variants here too. There's no tool to do
106 # the lossy things, so just use logger.
107 logger -t "ctdbd: ${_tag}" $*
108 ;;
109 esac
110 }
112 # When things are run in the background in an eventscript then logging
113 # output might get lost. This is the "solution". :-)
114 background_with_logging ()
115 {
116 (
117 "$@" 2>&1 </dev/null |
118 script_log "${script_name}&"
119 )&
121 return 0
122 }
124 ##############################################################
125 # check number of args for different events
126 ctdb_check_args ()
127 {
128 case "$1" in
129 takeip|releaseip)
130 if [ $# != 4 ]; then
131 echo "ERROR: must supply interface, IP and maskbits"
132 exit 1
133 fi
134 ;;
135 updateip)
136 if [ $# != 5 ]; then
137 echo "ERROR: must supply old interface, new interface, IP and maskbits"
138 exit 1
139 fi
140 ;;
141 esac
142 }
144 ##############################################################
145 # determine on what type of system (init style) we are running
146 detect_init_style()
147 {
148 # only do detection if not already set:
149 [ -z "$CTDB_INIT_STYLE" ] || return
151 if [ -x /sbin/startproc ]; then
152 CTDB_INIT_STYLE="suse"
153 elif [ -x /sbin/start-stop-daemon ]; then
154 CTDB_INIT_STYLE="debian"
155 else
156 CTDB_INIT_STYLE="redhat"
157 fi
158 }
160 ######################################################
161 # simulate /sbin/service on platforms that don't have it
162 # _service() makes it easier to hook the service() function for
163 # testing.
164 _service ()
165 {
166 _service_name="$1"
167 _op="$2"
169 # do nothing, when no service was specified
170 [ -z "$_service_name" ] && return
172 if [ -x /sbin/service ]; then
173 $_nice /sbin/service "$_service_name" "$_op"
174 elif [ -x /usr/sbin/service ]; then
175 $_nice /usr/sbin/service "$_service_name" "$_op"
176 elif [ -x $CTDB_ETCDIR/init.d/$_service_name ]; then
177 $_nice $CTDB_ETCDIR/init.d/$_service_name "$_op"
178 elif [ -x $CTDB_ETCDIR/rc.d/init.d/$_service_name ]; then
179 $_nice $CTDB_ETCDIR/rc.d/init.d/$_service_name "$_op"
180 fi
181 }
183 service()
184 {
185 _nice=""
186 _service "$@"
187 }
189 ######################################################
190 # simulate /sbin/service (niced) on platforms that don't have it
191 nice_service()
192 {
193 _nice="nice"
194 _service "$@"
195 }
197 ######################################################
198 # wrapper around /proc/ settings to allow them to be hooked
199 # for testing
200 # 1st arg is relative path under /proc/, 2nd arg is value to set
201 set_proc ()
202 {
203 echo "$2" >"/proc/$1"
204 }
206 ######################################################
207 # wrapper around getting file contents from /proc/ to allow
208 # this to be hooked for testing
209 # 1st arg is relative path under /proc/
210 get_proc ()
211 {
212 cat "/proc/$1"
213 }
215 ######################################################
216 # Print up to $_max kernel stack traces for processes named $_program
217 program_stack_traces ()
218 {
219 _prog="$1"
220 _max="${2:-1}"
222 _count=1
223 for _pid in $(pidof "$_prog") ; do
224 [ $_count -le $_max ] || break
226 # Do this first to avoid racing with process exit
227 _stack=$(get_proc "${_pid}/stack" 2>/dev/null)
228 if [ -n "$_stack" ] ; then
229 echo "Stack trace for ${_prog}[${_pid}]:"
230 echo "$_stack"
231 _count=$(($_count + 1))
232 fi
233 done
234 }
236 ######################################################
237 # Check that an RPC service is healthy -
238 # this includes allowing a certain number of failures
239 # before marking the NFS service unhealthy.
240 #
241 # usage: nfs_check_rpc_service SERVICE_NAME [ triple ...]
242 #
243 # each triple is a set of 3 arguments: an operator, a
244 # fail count limit and an action string.
245 #
246 # For example:
247 #
248 # nfs_check_rpc_service "lockd" \
249 # -ge 15 "verbose restart unhealthy" \
250 # -eq 10 "restart:bs"
251 #
252 # says that if lockd is down for 15 iterations then do
253 # a verbose restart of lockd and mark the node unhealthy.
254 # Before this, after 10 iterations of failure, the
255 # service is restarted silently in the background.
256 # Order is important: the number of failures need to be
257 # specified in reverse order because processing stops
258 # after the first condition that is true.
259 ######################################################
260 nfs_check_rpc_service ()
261 {
262 _prog_name="$1" ; shift
264 if _nfs_check_rpc_common "$_prog_name" ; then
265 return
266 fi
268 while [ -n "$3" ] ; do
269 if _nfs_check_rpc_action "$1" "$2" "$3" ; then
270 break
271 fi
272 shift 3
273 done
274 }
276 # The new way of doing things...
277 nfs_check_rpc_services ()
278 {
279 # Files must end with .check - avoids editor backups, RPM fu, ...
280 for _f in "${CTDB_BASE}/nfs-rpc-checks.d/"[0-9][0-9].*.check ; do
281 _t="${_f%.check}"
282 _prog_name="${_t##*/[0-9][0-9].}"
284 if _nfs_check_rpc_common "$_prog_name" ; then
285 # This RPC service is up, check next service...
286 continue
287 fi
289 # Check each line in the file in turn until one of the limit
290 # checks is hit...
291 while read _cmp _lim _rest ; do
292 # Skip comments
293 case "$_cmp" in
294 \#*) continue ;;
295 esac
297 if _nfs_check_rpc_action "$_cmp" "$_lim" "$_rest" ; then
298 # Limit was hit on this line, no further checking...
299 break
300 fi
301 done <"$_f"
302 done
303 }
305 _nfs_check_rpc_common ()
306 {
307 _prog_name="$1"
309 # Some platforms don't have separate programs for all services.
310 case "$_prog_name" in
311 statd)
312 which "rpc.${_prog_name}" >/dev/null 2>&1 || return 0
313 esac
315 case "$_prog_name" in
316 nfsd)
317 _rpc_prog=nfs
318 _version=3
319 ;;
320 mountd)
321 _rpc_prog=mountd
322 _version=1
323 ;;
324 rquotad)
325 _rpc_prog=rquotad
326 _version=1
327 ;;
328 lockd)
329 _rpc_prog=nlockmgr
330 _version=4
331 ;;
332 statd)
333 _rpc_prog=status
334 _version=1
335 ;;
336 *)
337 echo "Internal error: unknown RPC program \"$_prog_name\"."
338 exit 1
339 esac
341 _service_name="nfs_${_prog_name}"
343 if ctdb_check_rpc "$_rpc_prog" $_version >/dev/null ; then
344 ctdb_counter_init "$_service_name"
345 return 0
346 fi
348 ctdb_counter_incr "$_service_name"
350 return 1
351 }
353 _nfs_check_rpc_action ()
354 {
355 _cmp="$1"
356 _limit="$2"
357 _actions="$3"
359 if ctdb_check_counter "quiet" "$_cmp" "$_limit" "$_service_name" ; then
360 return 1
361 fi
363 for _action in $_actions ; do
364 case "$_action" in
365 verbose)
366 echo "$ctdb_check_rpc_out"
367 ;;
368 restart)
369 _nfs_restart_rpc_service "$_prog_name"
370 ;;
371 restart:b)
372 _nfs_restart_rpc_service "$_prog_name" true
373 ;;
374 unhealthy)
375 exit 1
376 ;;
377 *)
378 echo "Internal error: unknown action \"$_action\"."
379 exit 1
380 esac
381 done
383 return 0
384 }
386 _nfs_restart_rpc_service ()
387 {
388 _prog_name="$1"
389 _background="${2:-false}"
391 if $_background ; then
392 _maybe_background="background_with_logging"
393 else
394 _maybe_background=""
395 fi
397 _p="rpc.${_prog_name}"
399 case "$_prog_name" in
400 nfsd)
401 echo "Trying to restart NFS service"
402 $_maybe_background startstop_nfs restart
403 ;;
404 mountd)
405 echo "Trying to restart $_prog_name [${_p}]"
406 killall -q -9 "$_p"
407 nfs_dump_some_threads "$_p"
408 $_maybe_background $_p ${MOUNTD_PORT:+-p} $MOUNTD_PORT
409 ;;
410 rquotad)
411 echo "Trying to restart $_prog_name [${_p}]"
412 killall -q -9 "$_p"
413 nfs_dump_some_threads "$_p"
414 $_maybe_background $_p ${RQUOTAD_PORT:+-p} $RQUOTAD_PORT
415 ;;
416 lockd)
417 echo "Trying to restart lock manager service"
418 $_maybe_background startstop_nfslock restart
419 ;;
420 statd)
421 echo "Trying to restart $_prog_name [${_p}]"
422 killall -q -9 "$_p"
423 nfs_dump_some_threads "$_p"
424 $_maybe_background $_p \
425 ${STATD_HOSTNAME:+-n} $STATD_HOSTNAME \
426 ${STATD_PORT:+-p} $STATD_PORT \
427 ${STATD_OUTGOING_PORT:+-o} $STATD_OUTGOING_PORT
428 ;;
429 *)
430 echo "Internal error: unknown RPC program \"$_prog_name\"."
431 exit 1
432 esac
433 }
435 ######################################################
436 # check that a rpc server is registered with portmap
437 # and responding to requests
438 # usage: ctdb_check_rpc SERVICE_NAME VERSION
439 ######################################################
440 ctdb_check_rpc ()
441 {
442 progname="$1"
443 version="$2"
445 _localhost="${CTDB_RPCINFO_LOCALHOST:-127.0.0.1}"
447 if ! ctdb_check_rpc_out=$(rpcinfo -u $_localhost $progname $version 2>&1) ; then
448 ctdb_check_rpc_out="ERROR: $progname failed RPC check:
449 $ctdb_check_rpc_out"
450 echo "$ctdb_check_rpc_out"
451 return 1
452 fi
453 }
455 ######################################################
456 # Ensure $service_name is set
457 assert_service_name ()
458 {
459 [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
460 }
462 ######################################################
463 # check a set of directories is available
464 # return 1 on a missing directory
465 # directories are read from stdin
466 ######################################################
467 ctdb_check_directories_probe()
468 {
469 while IFS="" read d ; do
470 case "$d" in
471 *%*)
472 continue
473 ;;
474 *)
475 [ -d "${d}/." ] || return 1
476 esac
477 done
478 }
480 ######################################################
481 # check a set of directories is available
482 # directories are read from stdin
483 ######################################################
484 ctdb_check_directories()
485 {
486 ctdb_check_directories_probe || {
487 echo "ERROR: $service_name directory \"$d\" not available"
488 exit 1
489 }
490 }
492 ######################################################
493 # check a set of tcp ports
494 # usage: ctdb_check_tcp_ports <ports...>
495 ######################################################
497 # This flag file is created when a service is initially started. It
498 # is deleted the first time TCP port checks for that service succeed.
499 # Until then ctdb_check_tcp_ports() prints a more subtle "error"
500 # message if a port check fails.
501 _ctdb_check_tcp_common ()
502 {
503 assert_service_name
504 _ctdb_service_started_file="$ctdb_fail_dir/$service_name.started"
505 }
507 ctdb_check_tcp_init ()
508 {
509 _ctdb_check_tcp_common
510 mkdir -p "${_ctdb_service_started_file%/*}" # dirname
511 touch "$_ctdb_service_started_file"
512 }
514 # Check whether something is listening on all of the given TCP ports
515 # using the "ctdb checktcpport" command.
516 ctdb_check_tcp_ports()
517 {
518 if [ -z "$1" ] ; then
519 echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
520 exit 1
521 fi
523 for _p ; do # process each function argument (port)
524 _cmd="ctdb checktcpport $_p"
525 _out=$($_cmd 2>&1)
526 _ret=$?
527 case "$_ret" in
528 0)
529 _ctdb_check_tcp_common
530 if [ ! -f "$_ctdb_service_started_file" ] ; then
531 echo "ERROR: $service_name tcp port $_p is not responding"
532 debug "\"ctdb checktcpport $_p\" was able to bind to port"
533 else
534 echo "INFO: $service_name tcp port $_p is not responding"
535 fi
537 return 1
538 ;;
539 98)
540 # Couldn't bind, something already listening, next port...
541 continue
542 ;;
543 *)
544 echo "ERROR: unexpected error running \"ctdb checktcpport\""
545 debug <<EOF
546 ctdb checktcpport (exited with $_ret) with output:
547 $_out"
548 EOF
549 return $_ret
550 esac
551 done
553 # All ports listening
554 _ctdb_check_tcp_common
555 rm -f "$_ctdb_service_started_file"
556 return 0
557 }
559 ######################################################
560 # check a unix socket
561 # usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
562 ######################################################
563 ctdb_check_unix_socket() {
564 socket_path="$1"
565 [ -z "$socket_path" ] && return
567 if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
568 echo "ERROR: $service_name socket $socket_path not found"
569 return 1
570 fi
571 }
573 ######################################################
574 # check a command returns zero status
575 # usage: ctdb_check_command <command>
576 ######################################################
577 ctdb_check_command ()
578 {
579 _out=$("$@" 2>&1) || {
580 echo "ERROR: $* returned error"
581 echo "$_out" | debug
582 exit 1
583 }
584 }
586 ################################################
587 # kill off any TCP connections with the given IP
588 ################################################
589 kill_tcp_connections ()
590 {
591 _ip="$1"
593 _oneway=false
594 if [ "$2" = "oneway" ] ; then
595 _oneway=true
596 fi
598 get_tcp_connections_for_ip "$_ip" | {
599 _killcount=0
600 _connections=""
601 _nl="
602 "
603 while read _dst _src; do
604 _destport="${_dst##*:}"
605 __oneway=$_oneway
606 case $_destport in
607 # we only do one-way killtcp for CIFS
608 139|445) __oneway=true ;;
609 esac
611 echo "Killing TCP connection $_src $_dst"
612 _connections="${_connections}${_nl}${_src} ${_dst}"
613 if ! $__oneway ; then
614 _connections="${_connections}${_nl}${_dst} ${_src}"
615 fi
617 _killcount=$(($_killcount + 1))
618 done
620 if [ $_killcount -eq 0 ] ; then
621 return
622 fi
624 echo "$_connections" | ctdb killtcp || {
625 echo "Failed to send killtcp control"
626 return
627 }
629 _count=0
630 while : ; do
631 _remaining=$(get_tcp_connections_for_ip $_ip | wc -l)
633 if [ $_remaining -eq 0 ] ; then
634 echo "Killed $_killcount TCP connections to released IP $_ip"
635 return
636 fi
638 _count=$(($_count + 1))
639 if [ $_count -gt 3 ] ; then
640 echo "Timed out killing tcp connections for IP $_ip ($_remaining remaining)"
641 return
642 fi
644 echo "Waiting for $_remaining connections to be killed for IP $_ip"
645 sleep 1
646 done
647 }
648 }
650 ##################################################################
651 # kill off the local end for any TCP connections with the given IP
652 ##################################################################
653 kill_tcp_connections_local_only ()
654 {
655 kill_tcp_connections "$1" "oneway"
656 }
658 ##################################################################
659 # tickle any TCP connections with the given IP
660 ##################################################################
661 tickle_tcp_connections ()
662 {
663 _ip="$1"
665 get_tcp_connections_for_ip "$_ip" |
666 {
667 _failed=false
669 while read dest src; do
670 echo "Tickle TCP connection $src $dest"
671 ctdb tickle $src $dest >/dev/null 2>&1 || _failed=true
672 echo "Tickle TCP connection $dest $src"
673 ctdb tickle $dest $src >/dev/null 2>&1 || _failed=true
674 done
676 if $_failed ; then
677 echo "Failed to send tickle control"
678 fi
679 }
680 }
682 get_tcp_connections_for_ip ()
683 {
684 _ip="$1"
686 netstat -tn | awk -v ip=$_ip \
687 'index($1, "tcp") == 1 && \
688 (index($4, ip ":") == 1 || index($4, "::ffff:" ip ":") == 1) \
689 && $6 == "ESTABLISHED" \
690 {print $4" "$5}'
691 }
693 ##################################################################
694 # use statd-callout to update NFS lock info
695 ##################################################################
696 nfs_update_lock_info ()
697 {
698 if [ -x "$CTDB_BASE/statd-callout" ] ; then
699 "$CTDB_BASE/statd-callout" update
700 fi
701 }
703 ########################################################
704 # start/stop the Ganesha nfs service
705 ########################################################
706 startstop_ganesha()
707 {
708 _service_name="nfs-ganesha-$CTDB_CLUSTER_FILESYSTEM_TYPE"
709 case "$1" in
710 start)
711 service "$_service_name" start
712 ;;
713 stop)
714 service "$_service_name" stop
715 ;;
716 restart)
717 service "$_service_name" stop
718 nfs_dump_some_threads "rpc.statd"
719 service "$_service_name" start
720 ;;
721 esac
722 }
724 ########################################################
725 # start/stop the nfs service on different platforms
726 ########################################################
727 startstop_nfs() {
728 PLATFORM="unknown"
729 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
730 PLATFORM="sles"
731 }
732 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
733 -r /usr/lib/systemd/system/nfs-lock.service ] && {
734 PLATFORM="rhel"
735 }
737 case $PLATFORM in
738 sles)
739 case $1 in
740 start)
741 service nfsserver start
742 ;;
743 stop)
744 service nfsserver stop > /dev/null 2>&1
745 ;;
746 restart)
747 set_proc "fs/nfsd/threads" 0
748 service nfsserver stop > /dev/null 2>&1
749 pkill -9 nfsd
750 nfs_dump_some_threads
751 service nfsserver start
752 ;;
753 esac
754 ;;
755 rhel)
756 case $1 in
757 start)
758 service nfslock start
759 service nfs start
760 ;;
761 stop)
762 service nfs stop
763 service nfslock stop
764 ;;
765 restart)
766 set_proc "fs/nfsd/threads" 0
767 service nfs stop > /dev/null 2>&1
768 service nfslock stop > /dev/null 2>&1
769 pkill -9 nfsd
770 nfs_dump_some_threads
771 service nfslock start
772 service nfs start
773 ;;
774 esac
775 ;;
776 *)
777 echo "Unknown platform. NFS is not supported with ctdb"
778 exit 1
779 ;;
780 esac
781 }
783 # Dump up to the configured number of nfsd thread backtraces.
784 nfs_dump_some_threads ()
785 {
786 _prog="${1:-nfsd}"
788 _num="${CTDB_NFS_DUMP_STUCK_THREADS:-5}"
789 [ $_num -gt 0 ] || return 0
791 program_stack_traces "$_prog" $_num
792 }
794 ########################################################
795 # start/stop the nfs lockmanager service on different platforms
796 ########################################################
797 startstop_nfslock() {
798 PLATFORM="unknown"
799 [ -x $CTDB_ETCDIR/init.d/nfsserver ] && {
800 PLATFORM="sles"
801 }
802 [ -x $CTDB_ETCDIR/init.d/nfslock -o \
803 -r /usr/lib/systemd/system/nfs-lock.service ] && {
804 PLATFORM="rhel"
805 }
807 case $PLATFORM in
808 sles)
809 # for sles there is no service for lockmanager
810 # so we instead just shutdown/restart nfs
811 case $1 in
812 start)
813 service nfsserver start
814 ;;
815 stop)
816 service nfsserver stop > /dev/null 2>&1
817 ;;
818 restart)
819 service nfsserver stop > /dev/null 2>&1
820 service nfsserver start
821 ;;
822 esac
823 ;;
824 rhel)
825 case $1 in
826 start)
827 service nfslock start
828 ;;
829 stop)
830 service nfslock stop > /dev/null 2>&1
831 ;;
832 restart)
833 service nfslock stop > /dev/null 2>&1
834 service nfslock start
835 ;;
836 esac
837 ;;
838 *)
839 echo "Unknown platform. NFS locking is not supported with ctdb"
840 exit 1
841 ;;
842 esac
843 }
845 ########################################################
847 add_ip_to_iface ()
848 {
849 _iface=$1
850 _ip=$2
851 _maskbits=$3
853 # Ensure interface is up
854 ip link set "$_iface" up || \
855 die "Failed to bringup interface $_iface"
857 # Only need to define broadcast for IPv4
858 case "$ip" in
859 *:*) _bcast="" ;;
860 *) _bcast="brd +" ;;
861 esac
863 ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
864 echo "Failed to add $_ip/$_maskbits on dev $_iface"
865 return 1
866 }
868 # Wait 5 seconds for IPv6 addresses to stop being tentative...
869 if [ -z "$_bcast" ] ; then
870 for _x in $(seq 1 10) ; do
871 ip addr show to "${_ip}/128" | grep -q "tentative" || break
872 sleep 0.5
873 done
875 # If the address was a duplicate then it won't be on the
876 # interface so flag an error.
877 _t=$(ip addr show to "${_ip}/128")
878 case "$_t" in
879 "")
880 echo "Failed to add $_ip/$_maskbits on dev $_iface"
881 return 1
882 ;;
883 *tentative*|*dadfailed*)
884 echo "Failed to add $_ip/$_maskbits on dev $_iface"
885 ip addr del "$_ip/$_maskbits" dev "$_iface"
886 return 1
887 ;;
888 esac
889 fi
890 }
892 delete_ip_from_iface()
893 {
894 _iface=$1
895 _ip=$2
896 _maskbits=$3
898 # This could be set globally for all interfaces but it is probably
899 # better to avoid surprises, so limit it the interfaces where CTDB
900 # has public IP addresses. There isn't anywhere else convenient
901 # to do this so just set it each time. This is much cheaper than
902 # remembering and re-adding secondaries.
903 set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1
905 ip addr del "$_ip/$_maskbits" dev "$_iface" || {
906 echo "Failed to del $_ip on dev $_iface"
907 return 1
908 }
909 }
911 # If the given IP is hosted then print 2 items: maskbits and iface
912 ip_maskbits_iface ()
913 {
914 _addr="$1"
916 case "$_addr" in
917 *:*) _family="inet6" ; _bits=128 ;;
918 *) _family="inet" ; _bits=32 ;;
919 esac
921 ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
922 awk -v family="${_family}" \
923 'NR == 1 { iface = $2; sub(":$", "", iface) } \
924 $1 ~ /inet/ { mask = $2; sub(".*/", "", mask); \
925 print mask, iface, family }'
926 }
928 drop_ip ()
929 {
930 _addr="${1%/*}" # Remove optional maskbits
932 set -- $(ip_maskbits_iface $_addr)
933 if [ -n "$1" ] ; then
934 _maskbits="$1"
935 _iface="$2"
936 echo "Removing public address $_addr/$_maskbits from device $_iface"
937 delete_ip_from_iface $_iface $_addr $_maskbits >/dev/null 2>&1
938 fi
939 }
941 drop_all_public_ips ()
942 {
943 while read _ip _x ; do
944 drop_ip "$_ip"
945 done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
946 }
948 flush_route_cache ()
949 {
950 set_proc sys/net/ipv4/route/flush 1
951 set_proc sys/net/ipv6/route/flush 1
952 }
954 ########################################################
955 # Simple counters
956 _ctdb_counter_common () {
957 _service_name="${1:-${service_name:-${script_name}}}"
958 _counter_file="$ctdb_fail_dir/$_service_name"
959 mkdir -p "${_counter_file%/*}" # dirname
960 }
961 ctdb_counter_init () {
962 _ctdb_counter_common "$1"
964 >"$_counter_file"
965 }
966 ctdb_counter_incr () {
967 _ctdb_counter_common "$1"
969 # unary counting!
970 echo -n 1 >> "$_counter_file"
971 }
972 ctdb_check_counter () {
973 _msg="${1:-error}" # "error" - anything else is silent on fail
974 _op="${2:--ge}" # an integer operator supported by test
975 _limit="${3:-${service_fail_limit}}"
976 shift 3
977 _ctdb_counter_common "$1"
979 # unary counting!
980 _size=$(stat -c "%s" "$_counter_file" 2>/dev/null || echo 0)
981 _hit=false
982 if [ "$_op" != "%" ] ; then
983 if [ $_size $_op $_limit ] ; then
984 _hit=true
985 fi
986 else
987 if [ $(($_size $_op $_limit)) -eq 0 ] ; then
988 _hit=true
989 fi
990 fi
991 if $_hit ; then
992 if [ "$_msg" = "error" ] ; then
993 echo "ERROR: $_size consecutive failures for $_service_name, marking node unhealthy"
994 exit 1
995 else
996 return 1
997 fi
998 fi
999 }
1001 ########################################################
1003 ctdb_status_dir="$CTDB_VARDIR/state/service_status"
1004 ctdb_fail_dir="$CTDB_VARDIR/state/failcount"
1006 ctdb_setup_service_state_dir ()
1007 {
1008 service_state_dir="$CTDB_VARDIR/state/service_state/${1:-${service_name}}"
1009 mkdir -p "$service_state_dir" || {
1010 echo "Error creating state dir \"$service_state_dir\""
1011 exit 1
1012 }
1013 }
1015 ########################################################
1016 # Managed status history, for auto-start/stop
1018 ctdb_managed_dir="$CTDB_VARDIR/state/managed_history"
1020 _ctdb_managed_common ()
1021 {
1022 _ctdb_managed_file="$ctdb_managed_dir/$service_name"
1023 }
1025 ctdb_service_managed ()
1026 {
1027 _ctdb_managed_common
1028 mkdir -p "$ctdb_managed_dir"
1029 touch "$_ctdb_managed_file"
1030 }
1032 ctdb_service_unmanaged ()
1033 {
1034 _ctdb_managed_common
1035 rm -f "$_ctdb_managed_file"
1036 }
1038 is_ctdb_previously_managed_service ()
1039 {
1040 _ctdb_managed_common
1041 [ -f "$_ctdb_managed_file" ]
1042 }
1044 ########################################################
1045 # Check and set status
1047 log_status_cat ()
1048 {
1049 echo "node is \"$1\", \"${script_name}\" reports problem: $(cat $2)"
1050 }
1052 ctdb_checkstatus ()
1053 {
1054 if [ -r "$ctdb_status_dir/$script_name/unhealthy" ] ; then
1055 log_status_cat "unhealthy" "$ctdb_status_dir/$script_name/unhealthy"
1056 return 1
1057 elif [ -r "$ctdb_status_dir/$script_name/banned" ] ; then
1058 log_status_cat "banned" "$ctdb_status_dir/$script_name/banned"
1059 return 2
1060 else
1061 return 0
1062 fi
1063 }
1065 ctdb_setstatus ()
1066 {
1067 d="$ctdb_status_dir/$script_name"
1068 case "$1" in
1069 unhealthy|banned)
1070 mkdir -p "$d"
1071 cat "$2" >"$d/$1"
1072 ;;
1073 *)
1074 for i in "banned" "unhealthy" ; do
1075 rm -f "$d/$i"
1076 done
1077 ;;
1078 esac
1079 }
1081 ##################################################################
1082 # Reconfigure a service on demand
1084 _ctdb_service_reconfigure_common ()
1085 {
1086 _d="$ctdb_status_dir/${service_name}"
1087 mkdir -p "$_d"
1088 _ctdb_service_reconfigure_flag="$_d/reconfigure"
1089 }
1091 ctdb_service_needs_reconfigure ()
1092 {
1093 _ctdb_service_reconfigure_common
1094 [ -e "$_ctdb_service_reconfigure_flag" ]
1095 }
1097 ctdb_service_set_reconfigure ()
1098 {
1099 _ctdb_service_reconfigure_common
1100 >"$_ctdb_service_reconfigure_flag"
1101 }
1103 ctdb_service_unset_reconfigure ()
1104 {
1105 _ctdb_service_reconfigure_common
1106 rm -f "$_ctdb_service_reconfigure_flag"
1107 }
1109 ctdb_service_reconfigure ()
1110 {
1111 echo "Reconfiguring service \"${service_name}\"..."
1112 ctdb_service_unset_reconfigure
1113 service_reconfigure || return $?
1114 ctdb_counter_init
1115 }
1117 # Default service_reconfigure() function does nothing.
1118 service_reconfigure ()
1119 {
1120 :
1121 }
1123 ctdb_reconfigure_take_lock ()
1124 {
1125 _ctdb_service_reconfigure_common
1126 _lock="${_d}/reconfigure_lock"
1127 mkdir -p "${_lock%/*}" # dirname
1128 touch "$_lock"
1130 (
1131 flock 0
1132 # This is overkill but will work if we need to extend this to
1133 # allow certain events to run multiple times in parallel
1134 # (e.g. takeip) and write multiple PIDs to the file.
1135 read _locker_event
1136 if [ -n "$_locker_event" ] ; then
1137 while read _pid ; do
1138 if [ -n "$_pid" -a "$_pid" != $$ ] && \
1139 kill -0 "$_pid" 2>/dev/null ; then
1140 exit 1
1141 fi
1142 done
1143 fi
1145 printf "%s\n%s\n" "$event_name" $$ >"$_lock"
1146 exit 0
1147 ) <"$_lock"
1148 }
1150 ctdb_reconfigure_release_lock ()
1151 {
1152 _ctdb_service_reconfigure_common
1153 _lock="${_d}/reconfigure_lock"
1155 rm -f "$_lock"
1156 }
1158 ctdb_replay_monitor_status ()
1159 {
1160 echo "Replaying previous status for this script due to reconfigure..."
1161 # Leading separator ('|') is missing in some versions...
1162 _out=$(ctdb scriptstatus -X | grep -E "^\|?monitor\|${script_name}\|")
1163 # Output looks like this:
1164 # |monitor|60.nfs|1|ERROR|1314764004.030861|1314764004.035514|foo bar|
1165 # This is the cheapest way of getting fields in the middle.
1166 set -- $(IFS="|" ; echo $_out)
1167 _code="$3"
1168 _status="$4"
1169 # The error output field can include colons so we'll try to
1170 # preserve them. The weak checking at the beginning tries to make
1171 # this work for both broken (no leading '|') and fixed output.
1172 _out="${_out%|}"
1173 _err_out="${_out#*monitor|${script_name}|*|*|*|*|}"
1174 case "$_status" in
1175 OK) : ;; # Do nothing special.
1176 TIMEDOUT)
1177 # Recast this as an error, since we can't exit with the
1178 # correct negative number.
1179 _code=1
1180 _err_out="[Replay of TIMEDOUT scriptstatus - note incorrect return code.] ${_err_out}"
1181 ;;
1182 DISABLED)
1183 # Recast this as an OK, since we can't exit with the
1184 # correct negative number.
1185 _code=0
1186 _err_out="[Replay of DISABLED scriptstatus - note incorrect return code.] ${_err_out}"
1187 ;;
1188 *) : ;; # Must be ERROR, do nothing special.
1189 esac
1190 if [ -n "$_err_out" ] ; then
1191 echo "$_err_out"
1192 fi
1193 exit $_code
1194 }
1196 ctdb_service_check_reconfigure ()
1197 {
1198 assert_service_name
1200 # We only care about some events in this function. For others we
1201 # return now.
1202 case "$event_name" in
1203 monitor|ipreallocated|reconfigure) : ;;
1204 *) return 0 ;;
1205 esac
1207 if ctdb_reconfigure_take_lock ; then
1208 # No events covered by this function are running, so proceed
1209 # with gay abandon.
1210 case "$event_name" in
1211 reconfigure)
1212 (ctdb_service_reconfigure)
1213 exit $?
1214 ;;
1215 ipreallocated)
1216 if ctdb_service_needs_reconfigure ; then
1217 ctdb_service_reconfigure
1218 fi
1219 ;;
1220 esac
1222 ctdb_reconfigure_release_lock
1223 else
1224 # Somebody else is running an event we don't want to collide
1225 # with. We proceed with caution.
1226 case "$event_name" in
1227 reconfigure)
1228 # Tell whoever called us to retry.
1229 exit 2
1230 ;;
1231 ipreallocated)
1232 # Defer any scheduled reconfigure and just run the
1233 # rest of the ipreallocated event, as per the
1234 # eventscript. There's an assumption here that the
1235 # event doesn't depend on any scheduled reconfigure.
1236 # This is true in the current code.
1237 return 0
1238 ;;
1239 monitor)
1240 # There is most likely a reconfigure in progress so
1241 # the service is possibly unstable. As above, we
1242 # defer any scheduled reconfigured. We also replay
1243 # the previous monitor status since that's the best
1244 # information we have.
1245 ctdb_replay_monitor_status
1246 ;;
1247 esac
1248 fi
1249 }
1251 ##################################################################
1252 # Does CTDB manage this service? - and associated auto-start/stop
1254 ctdb_compat_managed_service ()
1255 {
1256 if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
1257 CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
1258 fi
1259 }
1261 is_ctdb_managed_service ()
1262 {
1263 assert_service_name
1265 # $t is used just for readability and to allow better accurate
1266 # matching via leading/trailing spaces
1267 t=" $CTDB_MANAGED_SERVICES "
1269 # Return 0 if "<space>$service_name<space>" appears in $t
1270 if [ "${t#* ${service_name} }" != "${t}" ] ; then
1271 return 0
1272 fi
1274 # If above didn't match then update $CTDB_MANAGED_SERVICES for
1275 # backward compatibility and try again.
1276 ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD" "vsftpd"
1277 ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA" "samba"
1278 ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND" "winbind"
1279 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "apache2"
1280 ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD" "httpd"
1281 ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI" "iscsi"
1282 ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD" "clamd"
1283 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs"
1284 ctdb_compat_managed_service "$CTDB_MANAGES_NFS" "nfs-ganesha-gpfs"
1286 t=" $CTDB_MANAGED_SERVICES "
1288 # Return 0 if "<space>$service_name<space>" appears in $t
1289 [ "${t#* ${service_name} }" != "${t}" ]
1290 }
1292 ctdb_start_stop_service ()
1293 {
1294 assert_service_name
1296 # Allow service-start/service-stop pseudo-events to start/stop
1297 # services when we're not auto-starting/stopping and we're not
1298 # monitoring.
1299 case "$event_name" in
1300 service-start)
1301 if is_ctdb_managed_service ; then
1302 die 'service-start event not permitted when service is managed'
1303 fi
1304 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1305 die 'service-start event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1306 fi
1307 ctdb_service_start
1308 exit $?
1309 ;;
1310 service-stop)
1311 if is_ctdb_managed_service ; then
1312 die 'service-stop event not permitted when service is managed'
1313 fi
1314 if [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] ; then
1315 die 'service-stop event not permitted with $CTDB_SERVICE_AUTOSTARTSTOP = yes'
1316 fi
1317 ctdb_service_stop
1318 exit $?
1319 ;;
1320 esac
1322 # Do nothing unless configured to...
1323 [ "$CTDB_SERVICE_AUTOSTARTSTOP" = "yes" ] || return 0
1325 [ "$event_name" = "monitor" ] || return 0
1327 if is_ctdb_managed_service ; then
1328 if ! is_ctdb_previously_managed_service ; then
1329 echo "Starting service \"$service_name\" - now managed"
1330 background_with_logging ctdb_service_start
1331 exit $?
1332 fi
1333 else
1334 if is_ctdb_previously_managed_service ; then
1335 echo "Stopping service \"$service_name\" - no longer managed"
1336 background_with_logging ctdb_service_stop
1337 exit $?
1338 fi
1339 fi
1340 }
1342 ctdb_service_start ()
1343 {
1344 # The service is marked managed if we've ever tried to start it.
1345 ctdb_service_managed
1347 service_start || return $?
1349 ctdb_counter_init
1350 ctdb_check_tcp_init
1351 }
1353 ctdb_service_stop ()
1354 {
1355 ctdb_service_unmanaged
1356 service_stop
1357 }
1359 # Default service_start() and service_stop() functions.
1361 # These may be overridden in an eventscript.
1362 service_start ()
1363 {
1364 service "$service_name" start
1365 }
1367 service_stop ()
1368 {
1369 service "$service_name" stop
1370 }
1372 ##################################################################
1374 ctdb_standard_event_handler ()
1375 {
1376 case "$1" in
1377 status)
1378 ctdb_checkstatus
1379 exit
1380 ;;
1381 setstatus)
1382 shift
1383 ctdb_setstatus "$@"
1384 exit
1385 ;;
1386 esac
1387 }
1389 iptables_wrapper ()
1390 {
1391 _family="$1" ; shift
1392 if [ "$_family" = "inet6" ] ; then
1393 _iptables_cmd="ip6tables"
1394 else
1395 _iptables_cmd="iptables"
1396 fi
1398 # iptables doesn't like being re-entered, so flock-wrap it.
1399 flock -w 30 "${CTDB_VARDIR}/iptables-ctdb.flock" "$_iptables_cmd" "$@"
1400 }
1402 # AIX (and perhaps others?) doesn't have mktemp
1403 if ! which mktemp >/dev/null 2>&1 ; then
1404 mktemp ()
1405 {
1406 _dir=false
1407 if [ "$1" = "-d" ] ; then
1408 _dir=true
1409 shift
1410 fi
1411 _d="${TMPDIR:-/tmp}"
1412 _hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
1413 md5sum | \
1414 sed -e 's@\(..........\).*@\1@')
1415 _t="${_d}/tmp.${_hex10}"
1416 (
1417 umask 077
1418 if $_dir ; then
1419 mkdir "$_t"
1420 else
1421 >"$_t"
1422 fi
1423 )
1424 echo "$_t"
1425 }
1426 fi
1428 ########################################################
1429 # tickle handling
1430 ########################################################
1432 update_tickles ()
1433 {
1434 _port="$1"
1436 tickledir="$CTDB_VARDIR/state/tickles"
1437 mkdir -p "$tickledir"
1439 # Who am I?
1440 _pnn=$(ctdb pnn) ; _pnn=${_pnn#PNN:}
1442 # What public IPs do I hold?
1443 _ips=$(ctdb -X ip | awk -F'|' -v pnn=$_pnn '$3 == pnn {print $2}')
1445 # IPs as a regexp choice
1446 _ipschoice="($(echo $_ips | sed -e 's/ /|/g' -e 's/\./\\\\./g'))"
1448 # Record connections to our public IPs in a temporary file
1449 _my_connections="${tickledir}/${_port}.connections"
1450 rm -f "$_my_connections"
1451 netstat -tn |
1452 awk -v destpat="^${_ipschoice}:${_port}\$" \
1453 '$1 == "tcp" && $6 == "ESTABLISHED" && $4 ~ destpat {print $5, $4}' |
1454 sort >"$_my_connections"
1456 # Record our current tickles in a temporary file
1457 _my_tickles="${tickledir}/${_port}.tickles"
1458 rm -f "$_my_tickles"
1459 for _i in $_ips ; do
1460 ctdb -X gettickles $_i $_port |
1461 awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
1462 done |
1463 sort >"$_my_tickles"
1465 # Add tickles for connections that we haven't already got tickles for
1466 comm -23 "$_my_connections" "$_my_tickles" |
1467 while read _src _dst ; do
1468 ctdb addtickle $_src $_dst
1469 done
1471 # Remove tickles for connections that are no longer there
1472 comm -13 "$_my_connections" "$_my_tickles" |
1473 while read _src _dst ; do
1474 ctdb deltickle $_src $_dst
1475 done
1477 rm -f "$_my_connections" "$_my_tickles"
1478 }
1480 ########################################################
1481 # load a site local config file
1482 ########################################################
1484 [ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
1485 . "$CTDB_RC_LOCAL"
1486 }
1488 [ -x $CTDB_BASE/rc.local ] && {
1489 . $CTDB_BASE/rc.local
1490 }
1492 [ -d $CTDB_BASE/rc.local.d ] && {
1493 for i in $CTDB_BASE/rc.local.d/* ; do
1494 [ -x "$i" ] && . "$i"
1495 done
1496 }
1498 script_name="${0##*/}" # basename
1499 service_fail_limit=1
1500 event_name="$1"